Construct verkle execution witness (#3864)

* Update historyStorage address

* Remove execution prestate since unused

* Finish generateExecutionWitness function

* execution witness fixes

* Fix logic in building suffix diffs

* Add test

* spell check

* Address feedback

* Lint

* add lock and update comments

---------

Co-authored-by: Gabriel Rocheleau <[email protected]>

Author: acolytec3

package-lock.json

@@ -49,6 +49,7 @@
     "@ethereumjs/common": "^5.0.0-alpha.1",
     "@ethereumjs/statemanager": "^3.0.0-alpha.1",
     "@ethereumjs/util": "^10.0.0-alpha.1",
+    "@ethereumjs/verkle": "^0.2.0-alpha.1",
     "@noble/curves": "^1.8.1",
     "@types/debug": "^4.1.9",
     "debug": "^4.3.3",

packages/evm/package.json

@@ -271,7 +271,7 @@ export const paramsEVM: ParamsDict = {
    */
   2935: {
     // evm
-    historyStorageAddress: '0x0aae40965e6800cd9b1f4b05ff21581047e3f91e', // The address where the historical blockhashes are stored
+    historyStorageAddress: '0x0000F90827F1C53A10CB7A02335B175320002935', // The address where the historical blockhashes are stored
     historyServeWindow: 8192, // The amount of blocks to be served by the historical blockhash contract
     systemAddress: '0xfffffffffffffffffffffffffffffffffffffffe', // The system address
   },

packages/evm/src/params.ts

@@ -8,10 +8,12 @@ import {
   VERKLE_MAIN_STORAGE_OFFSET,
   VERKLE_NODE_WIDTH,
   bytesToHex,
+  equalsBytes,
   getVerkleKey,
   getVerkleStem,
   getVerkleTreeIndicesForCodeChunk,
   getVerkleTreeIndicesForStorageSlot,
+  hexToBytes,
   intToBytes,
 } from '@ethereumjs/util'
 import debugDefault from 'debug'
@@ -26,7 +28,14 @@ import type {
   VerkleAccessedState,
   VerkleAccessedStateWithAddress,
 } from '@ethereumjs/common'
-import type { Address, PrefixedHexString, VerkleCrypto } from '@ethereumjs/util'
+import type { StatefulVerkleStateManager } from '@ethereumjs/statemanager'
+import type {
+  Address,
+  PrefixedHexString,
+  VerkleCrypto,
+  VerkleExecutionWitness,
+} from '@ethereumjs/util'
+import type { VerkleTree } from '@ethereumjs/verkle'
 
 const debug = debugDefault('evm:verkle:aw')
 
@@ -377,3 +386,70 @@ export class VerkleAccessWitness implements VerkleAccessWitnessInterface {
     }
   }
 }
+
+/**
+ * Generate a {@link VerkleExecutionWitness} from a state manager and an access witness.
+ * @param stateManager - The state manager containing the state to generate the witness for.
+ * @param accessWitness - The access witness containing the accessed states.
+ * @param parentStateRoot - The parent state root (i.e. prestate root) to generate the witness for.
+ * @returns The generated verkle execution witness
+ *
+ * Note: This does not provide the verkle proof, which is not implemented
+ */
+export const generateExecutionWitness = async (
+  stateManager: StatefulVerkleStateManager,
+  accessWitness: VerkleAccessWitness,
+  parentStateRoot: Uint8Array,
+): Promise<VerkleExecutionWitness> => {
+  const trie = stateManager['_trie'] as VerkleTree
+  await trie['_lock'].acquire()
+  const postStateRoot = await stateManager.getStateRoot()
+  const ew: VerkleExecutionWitness = {
+    stateDiff: [],
+    parentStateRoot: bytesToHex(parentStateRoot),
+    verkleProof: undefined as any, // Verkle proofs are not implemented (and never will be)
+  }
+
+  // Generate a map of all stems with their accessed suffixes
+  const accessedSuffixes = new Map<PrefixedHexString, number[]>()
+  for (const chunkKey of accessWitness['chunks'].keys()) {
+    const stem = chunkKey.slice(0, 64) as PrefixedHexString
+    if (accessedSuffixes.has(stem)) {
+      const suffixes = accessedSuffixes.get(stem)
+      suffixes!.push(parseInt(chunkKey.slice(64), 16))
+      accessedSuffixes.set(stem, suffixes!)
+    } else {
+      accessedSuffixes.set(stem, [parseInt(chunkKey.slice(64), 16)])
+    }
+  }
+
+  // Get values from the trie for each stem and suffix
+  for (const stem of accessedSuffixes.keys()) {
+    trie.root(parentStateRoot)
+    const suffixes = accessedSuffixes.get(stem)
+    if (suffixes === undefined || suffixes.length === 0) continue
+    const currentValues = await trie.get(hexToBytes(stem), suffixes)
+    trie.root(postStateRoot)
+    const newValues = await trie.get(hexToBytes(stem), suffixes)
+    const stemStateDiff = []
+    for (let x = 0; x < suffixes.length; x++) {
+      // skip if both are the same
+      const currentValue = currentValues[x]
+      const newValue = newValues[x]
+      if (
+        currentValue instanceof Uint8Array &&
+        newValue instanceof Uint8Array &&
+        equalsBytes(currentValue, newValue)
+      )
+        continue
+      stemStateDiff.push({
+        suffix: suffixes[x],
+        currentValue: currentValue ? bytesToHex(currentValue) : null,
+        newValue: newValue ? bytesToHex(newValue) : null,
+      })
+    }
+    ew.stateDiff.push({ stem, suffixDiffs: stemStateDiff })
+  }
+  trie['_lock'].release()
+  return ew
+}

packages/evm/src/verkleAccessWitness.ts

@@ -2,16 +2,19 @@ import { Common, Hardfork, Mainnet } from '@ethereumjs/common'
 import { StatefulVerkleStateManager } from '@ethereumjs/statemanager'
 import {
   bigIntToBytes,
+  bytesToHex,
   createAccount,
   createAddressFromString,
+  decodeVerkleLeafBasicData,
+  getVerkleStem,
   hexToBytes,
   setLengthLeft,
 } from '@ethereumjs/util'
 import { createVerkleTree } from '@ethereumjs/verkle'
 import * as verkle from 'micro-eth-signer/verkle'
 import { assert, describe, it } from 'vitest'
 
-import { VerkleAccessWitness, createEVM } from '../src/index.js'
+import { VerkleAccessWitness, createEVM, generateExecutionWitness } from '../src/index.js'
 
 describe('verkle tests', () => {
   it('should execute bytecode and update the state', async () => {
@@ -109,3 +112,79 @@ describe('verkle tests', () => {
     assert.equal(res.execResult.exceptionError?.error, undefined)
   })
 })
+describe('generate an execution witness', () => {
+  it('should generate the correct execution witness from a prestate and changes', async () => {
+    const preStateVKT = {
+      '0x0365b079a274a1808d56484ce5bd97914629907d75767f51439102e22cd50d00':
+        '0x00000000000000000000000000000000000000000000003635c9adc5dea00000',
+      '0x0365b079a274a1808d56484ce5bd97914629907d75767f51439102e22cd50d01':
+        '0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470',
+      '0x5b5fdfedd6a0e932da408ac7d772a36513d1eee9b9926e52620c43a433aad700':
+        '0x0000000000000036000000000000000100000000000000000000000000000000',
+      '0x5b5fdfedd6a0e932da408ac7d772a36513d1eee9b9926e52620c43a433aad701':
+        '0xdf61faef43babbb1ebde8fd82ab9cb4cb74c240d0025138521477e073f72080a',
+      '0x5b5fdfedd6a0e932da408ac7d772a36513d1eee9b9926e52620c43a433aad780':
+        '0x0060203611603157600143035f35116029575f35612000014311602957612000',
+      '0x5b5fdfedd6a0e932da408ac7d772a36513d1eee9b9926e52620c43a433aad781':
+        '0x005f3506545f5260205ff35b5f5f5260205ff35b5f5ffd000000000000000000',
+    }
+    const tx = {
+      type: '0x0',
+      chainId: '0x1',
+      nonce: '0x0',
+      gasPrice: '0xa',
+      gas: '0x5f5e100',
+      to: '0x8a0a19589531694250d570040a0c4b74576919b8',
+      value: '0x0',
+      input: '0x',
+      v: '0x25',
+      r: '0x50ae258f0b1f7c44e5227b43c338aa7f2d9805115b90a6baeaaee2358796e074',
+      s: '0xec910ad0244580c17e1d6a512b3574c62e92840184109e3037760d39b20cb94',
+      sender: '0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b',
+    }
+
+    const common = new Common({
+      chain: Mainnet,
+      customCrypto: { verkle },
+      eips: [6800],
+      hardfork: Hardfork.Prague,
+    })
+    const trie = await createVerkleTree()
+    // Setup prestate
+    for (const [key, value] of Object.entries(preStateVKT)) {
+      const stem = hexToBytes(key).slice(0, 31)
+      const suffix = parseInt(key.slice(64), 16)
+      await trie.put(stem, [suffix], [hexToBytes(value)])
+    }
+    const preStateRoot = trie.root()
+    const sm = new StatefulVerkleStateManager({ common, trie })
+    const evm = await createEVM({ common, stateManager: sm })
+    evm.verkleAccessWitness = new VerkleAccessWitness({
+      verkleCrypto: verkle,
+    })
+    evm.systemVerkleAccessWitness = new VerkleAccessWitness({
+      verkleCrypto: verkle,
+    })
+    // Run tx
+    await evm.runCall({
+      code: hexToBytes(tx.input),
+      caller: createAddressFromString(tx.sender),
+      to: createAddressFromString(tx.to),
+      gasLimit: BigInt(tx.gas),
+      gasPrice: BigInt(tx.gasPrice),
+    })
+    const executionWitness = await generateExecutionWitness(
+      sm,
+      evm.verkleAccessWitness,
+      preStateRoot,
+    )
+    const stem = bytesToHex(getVerkleStem(verkle, createAddressFromString(tx.sender)))
+    assert.ok(executionWitness.stateDiff.findIndex((diff) => diff.stem === stem) !== -1)
+    const stemDiff =
+      executionWitness.stateDiff[executionWitness.stateDiff.findIndex((diff) => diff.stem === stem)]
+    const suffixDiff = stemDiff.suffixDiffs.find((diff) => diff.suffix === 0)
+    assert.ok(suffixDiff?.newValue !== undefined)
+    // Ensure sender account nonce is 1 in execution witness
+    assert.equal(decodeVerkleLeafBasicData(hexToBytes(suffixDiff!.newValue!)).nonce, 1n)
+  })
+})

packages/evm/test/verkle.spec.ts

@@ -63,6 +63,7 @@ export class StatefulVerkleStateManager implements StateManagerInterface {
   protected _debug: Debugger
   protected _caches?: Caches
 
+  preStateRoot: Uint8Array
   originalStorageCache: OriginalStorageCache
   verkleCrypto: VerkleCrypto
 
@@ -75,7 +76,6 @@ export class StatefulVerkleStateManager implements StateManagerInterface {
   // Post-state provided from the executionWitness.
   // Should not update. Used for comparing our computed post-state with the canonical one.
   private _postState: VerkleState = {}
-  private _preState: VerkleState = {}
 
   /**
    * StateManager is run in DEBUG mode (default: false)
@@ -88,6 +88,7 @@ export class StatefulVerkleStateManager implements StateManagerInterface {
   protected readonly DEBUG: boolean = false
 
   private keccakFunction: Function
+
   constructor(opts: StatefulVerkleStateManagerOpts) {
     // Skip DEBUG calls unless 'ethjs' included in environmental DEBUG variables
     // Additional window check is to prevent vite browser bundling (and potentially other) to break
@@ -118,6 +119,7 @@ export class StatefulVerkleStateManager implements StateManagerInterface {
     this._caches = opts.caches
     this.keccakFunction = opts.common.customCrypto.keccak256 ?? keccak256
     this.verkleCrypto = opts.common.customCrypto.verkle
+    this.preStateRoot = new Uint8Array(32) // Initial state root is zeroes
   }
 
   /**
@@ -179,23 +181,9 @@ export class StatefulVerkleStateManager implements StateManagerInterface {
       throw Error(errorMsg)
     }
 
-    // Populate the pre-state and post-state from the executionWitness
-    const preStateRaw = executionWitness.stateDiff.flatMap(({ stem, suffixDiffs }) => {
-      const suffixDiffPairs = suffixDiffs.map(({ currentValue, suffix }) => {
-        const key = `${stem}${padToEven(Number(suffix).toString(16))}`
-        return {
-          [key]: currentValue,
-        }
-      })
-
-      return suffixDiffPairs
-    })
+    this.preStateRoot = hexToBytes(executionWitness.parentStateRoot) // set prestate root if given
 
-    // also maintain a separate preState unaffected by any changes in _state
-    this._preState = preStateRaw.reduce((prevValue, currentValue) => {
-      const acc = { ...prevValue, ...currentValue }
-      return acc
-    }, {})
+    // Populate the post-state from the executionWitness
 
     const postStateRaw = executionWitness.stateDiff.flatMap(({ stem, suffixDiffs }) => {
       const suffixDiffPairs = suffixDiffs.map(({ newValue, currentValue, suffix }) => {
@@ -219,7 +207,6 @@ export class StatefulVerkleStateManager implements StateManagerInterface {
 
     this._postState = postState
 
-    this._debug(`initVerkleExecutionWitness preState=${JSON.stringify(this._preState)}`)
     this._debug(`initVerkleExecutionWitness postState=${JSON.stringify(this._postState)}`)
   }
 

packages/statemanager/src/statefulVerkleStateManager.ts

@@ -70,13 +70,6 @@ export const paramsVM: ParamsDict = {
     // config
     historicalRootsLength: 8191, // The modulo parameter of the beaconroot ring buffer in the beaconroot stateful precompile
   },
-  /**
-   * Ethereum state using a unified verkle tree (experimental)
-   */
-  6800: {
-    // config
-    historyStorageAddress: '0x0aae40965e6800cd9b1f4b05ff21581047e3f91e', // The address where the historical blockhashes are stored
-  },
   /**
    * Execution layer triggerable withdrawals (experimental)
    */