Client: separate TxIndex from ReceiptsManager (#4012)

* client: create txIndex.ts and export index types

* client: extract txIndex logic from receiptsManager to txIndex

* client: add txIndex as attribute of vmexecution

* client: move calls to updateIndex out of receiptsManager

* client: update RPC methods to use txIndex

* test: use MemoryLevel in test

* test: fix receipts test / add txIndex tests

* client: remove unnecessary lookup from eth_getTransactionByHash

* optimize debug_getRawTransaction

* test TxIndex class

* test vm exec initialization with metaDB

* remove unreachable case

* test get transaction without index

---------

Co-authored-by: acolytec3 <[email protected]>

Author: ScottyPoi

packages/client/src/execution/receipt.ts

@@ -1,6 +1,5 @@
 import { RLP } from '@ethereumjs/rlp'
 import {
-  BIGINT_0,
   EthereumJSErrorWithoutCode,
   bigIntToBytes,
   bytesToBigInt,
@@ -17,6 +16,7 @@ import type { Block } from '@ethereumjs/block'
 import type { Log } from '@ethereumjs/evm'
 import type { TransactionType, TypedTransaction } from '@ethereumjs/tx'
 import type { PostByzantiumTxReceipt, PreByzantiumTxReceipt, TxReceipt } from '@ethereumjs/vm'
+import type { TxHashIndex } from './txIndex.ts'
 
 /**
  * TxReceiptWithType extends TxReceipt to provide:
@@ -49,30 +49,11 @@ type GetLogsReturn = {
   logIndex: number
 }[]
 
-/**
- * Indexes
- */
-type TxHashIndex = [blockHash: Uint8Array, txIndex: number]
-
-export type IndexType = (typeof IndexType)[keyof typeof IndexType]
-
-export const IndexType = {
-  TxHash: 'txhash',
-} as const
-
-export type IndexOperation = (typeof IndexOperation)[keyof typeof IndexOperation]
-
-export const IndexOperation = {
-  Save: 'save',
-  Delete: 'delete',
-} as const
-
 /**
  * Storage encodings
  */
 type rlpLog = Log
 type rlpReceipt = [postStateOrStatus: Uint8Array, cumulativeGasUsed: Uint8Array, logs: rlpLog[]]
-type rlpTxHash = [blockHash: Uint8Array, txIndex: Uint8Array]
 
 export type RlpConvert = (typeof RlpConvert)[keyof typeof RlpConvert]
 
@@ -85,9 +66,8 @@ export type RlpType = (typeof RlpType)[keyof typeof RlpType]
 export const RlpType = {
   Receipts: 'receipts',
   Logs: 'logs',
-  TxHash: 'txhash',
 } as const
-type rlpOut = Log[] | TxReceipt[] | TxHashIndex
+type rlpOut = Log[] | TxReceipt[]
 
 export class ReceiptsManager extends MetaDBManager {
   /**
@@ -114,12 +94,10 @@ export class ReceiptsManager extends MetaDBManager {
   async saveReceipts(block: Block, receipts: TxReceipt[]) {
     const encoded = this.rlp(RlpConvert.Encode, RlpType.Receipts, receipts)
     await this.put(DBKey.Receipts, block.hash(), encoded)
-    void this.updateIndex(IndexOperation.Save, IndexType.TxHash, block)
   }
 
   async deleteReceipts(block: Block) {
     await this.delete(DBKey.Receipts, block.hash())
-    void this.updateIndex(IndexOperation.Delete, IndexType.TxHash, block)
   }
 
   /**
@@ -166,9 +144,9 @@ export class ReceiptsManager extends MetaDBManager {
    * Returns receipt by tx hash with additional metadata for the JSON RPC response, or null if not found
    * @param txHash the tx hash
    */
-  async getReceiptByTxHash(txHash: Uint8Array): Promise<GetReceiptByTxHashReturn | null> {
-    const txHashIndex = await this.getIndex(IndexType.TxHash, txHash)
-    if (!txHashIndex) return null
+  async getReceiptByTxHashIndex(
+    txHashIndex: TxHashIndex,
+  ): Promise<GetReceiptByTxHashReturn | null> {
     const [blockHash, txIndex] = txHashIndex
     const receipts = await this.getReceipts(blockHash)
     if (receipts.length === 0) return null
@@ -247,72 +225,6 @@ export class ReceiptsManager extends MetaDBManager {
     return returnedLogs
   }
 
-  /**
-   * Saves or deletes an index from the metaDB
-   * @param operation the {@link IndexOperation}
-   * @param type the {@link IndexType}
-   * @param value for {@link IndexType.TxHash}, the block to save or delete the tx hash indexes for
-   */
-  private async updateIndex(
-    operation: IndexOperation,
-    type: typeof IndexType.TxHash,
-    value: Block,
-  ): Promise<void>
-  private async updateIndex(operation: IndexOperation, type: IndexType, value: any): Promise<void> {
-    switch (type) {
-      case IndexType.TxHash: {
-        const block = value
-        if (operation === IndexOperation.Save) {
-          const withinTxLookupLimit =
-            this.config.txLookupLimit === 0 ||
-            this.chain.headers.height - BigInt(this.config.txLookupLimit) < block.header.number
-          if (withinTxLookupLimit) {
-            for (const [i, tx] of block.transactions.entries()) {
-              const index: TxHashIndex = [block.hash(), i]
-              const encoded = this.rlp(RlpConvert.Encode, RlpType.TxHash, index)
-              await this.put(DBKey.TxHash, tx.hash(), encoded)
-            }
-          }
-          if (this.config.txLookupLimit > 0) {
-            // Remove tx hashes for one block past txLookupLimit
-            const limit = this.chain.headers.height - BigInt(this.config.txLookupLimit)
-            if (limit < BIGINT_0) return
-            const blockDelIndexes = await this.chain.getBlock(limit)
-            void this.updateIndex(IndexOperation.Delete, IndexType.TxHash, blockDelIndexes)
-          }
-        } else if (operation === IndexOperation.Delete) {
-          for (const tx of block.transactions) {
-            await this.delete(DBKey.TxHash, tx.hash())
-          }
-        }
-        break
-      }
-      default:
-        throw EthereumJSErrorWithoutCode('Unsupported index type')
-    }
-  }
-
-  /**
-   * Returns the value for an index or null if not found
-   * @param type the {@link IndexType}
-   * @param value for {@link IndexType.TxHash}, the txHash to get
-   */
-  private async getIndex(
-    type: typeof IndexType.TxHash,
-    value: Uint8Array,
-  ): Promise<TxHashIndex | null>
-  private async getIndex(type: IndexType, value: Uint8Array): Promise<any | null> {
-    switch (type) {
-      case IndexType.TxHash: {
-        const encoded = await this.get(DBKey.TxHash, value)
-        if (!encoded) return null
-        return this.rlp(RlpConvert.Decode, RlpType.TxHash, encoded)
-      }
-      default:
-        throw EthereumJSErrorWithoutCode('Unsupported index type')
-    }
-  }
-
   /**
    * RLP encodes or decodes the specified data type for storage or retrieval from the metaDB
    * @param conversion {@link RlpConvert.Encode} or {@link RlpConvert.Decode}
@@ -330,11 +242,6 @@ export class ReceiptsManager extends MetaDBManager {
     type: typeof RlpType.Logs,
     value: rlpLog[],
   ): Log[]
-  private rlp(
-    conversion: typeof RlpConvert.Decode,
-    type: typeof RlpType.TxHash,
-    value: Uint8Array,
-  ): TxHashIndex
   private rlp(
     conversion: RlpConvert,
     type: RlpType,
@@ -380,14 +287,6 @@ export class ReceiptsManager extends MetaDBManager {
         } else {
           return RLP.decode(value as Uint8Array) as Log[]
         }
-      case RlpType.TxHash:
-        if (conversion === RlpConvert.Encode) {
-          const [blockHash, txIndex] = value as TxHashIndex
-          return RLP.encode([blockHash, intToBytes(txIndex)])
-        } else {
-          const [blockHash, txIndex] = RLP.decode(value as Uint8Array) as unknown as rlpTxHash
-          return [blockHash, bytesToInt(txIndex)] as TxHashIndex
-        }
       default:
         throw EthereumJSErrorWithoutCode('Unknown rlp conversion')
     }

packages/client/src/execution/txIndex.ts

@@ -0,0 +1,71 @@
+import { RLP } from '@ethereumjs/rlp'
+import { BIGINT_0, bytesToInt, intToBytes } from '@ethereumjs/util'
+import { DBKey, MetaDBManager } from '../util/metaDBManager.ts'
+
+export type TxHashIndex = [blockHash: Uint8Array, txIndex: number]
+
+export type IndexType = (typeof IndexType)[keyof typeof IndexType]
+
+export const IndexType = {
+  TxHash: 'txhash',
+} as const
+
+export type IndexOperation = (typeof IndexOperation)[keyof typeof IndexOperation]
+
+export const IndexOperation = {
+  Save: 'save',
+  Delete: 'delete',
+} as const
+
+export type rlpTxHash = [blockHash: Uint8Array, txIndex: Uint8Array]
+export class TxIndex extends MetaDBManager {
+  private rlpEncode(value: TxHashIndex): Uint8Array {
+    const [blockHash, txIndex] = value as TxHashIndex
+    return RLP.encode([blockHash, intToBytes(txIndex)])
+  }
+  private rlpDecode(value: Uint8Array): TxHashIndex {
+    const [blockHash, txIndex] = RLP.decode(value) as unknown as rlpTxHash
+    return [blockHash, bytesToInt(txIndex)] as TxHashIndex
+  }
+  async updateIndex(operation: IndexOperation, type: IndexType, value: any): Promise<void> {
+    switch (type) {
+      case IndexType.TxHash: {
+        const block = value
+        if (operation === IndexOperation.Save) {
+          const withinTxLookupLimit =
+            this.config.txLookupLimit === 0 ||
+            this.chain.headers.height - BigInt(this.config.txLookupLimit) < block.header.number
+          if (withinTxLookupLimit) {
+            for (const [i, tx] of block.transactions.entries()) {
+              const index: TxHashIndex = [block.hash(), i]
+              const encoded = this.rlpEncode(index)
+              await this.put(DBKey.TxHash, tx.hash(), encoded)
+            }
+          }
+          if (this.config.txLookupLimit > 0) {
+            // Remove tx hashes for one block past txLookupLimit
+            const limit = this.chain.headers.height - BigInt(this.config.txLookupLimit)
+            if (limit < BIGINT_0) return
+            const blockDelIndexes = await this.chain.getBlock(limit)
+            void this.updateIndex(IndexOperation.Delete, IndexType.TxHash, blockDelIndexes)
+          }
+        } else if (operation === IndexOperation.Delete) {
+          for (const tx of block.transactions) {
+            await this.delete(DBKey.TxHash, tx.hash())
+          }
+        }
+        break
+      }
+    }
+  }
+
+  /**
+   * Returns the value for an index or null if not found
+   * @param value for {@link IndexType.TxHash}, the txHash to get
+   */
+  async getIndex(value: Uint8Array): Promise<TxHashIndex | null> {
+    const encoded = await this.get(DBKey.TxHash, value)
+    if (encoded === null) return null
+    return this.rlpDecode(encoded)
+  }
+}

packages/client/src/execution/vmexecution.ts

@@ -38,6 +38,7 @@ import { Execution } from './execution.ts'
 import { LevelDB } from './level.ts'
 import { PreimagesManager } from './preimage.ts'
 import { ReceiptsManager } from './receipt.ts'
+import { IndexOperation, IndexType, TxIndex } from './txIndex.ts'
 
 import type { Block } from '@ethereumjs/block'
 import type { PrefixedHexString } from '@ethereumjs/util'
@@ -71,6 +72,7 @@ export class VMExecution extends Execution {
 
   public receiptsManager?: ReceiptsManager
   public preimagesManager?: PreimagesManager
+  public txIndex?: TxIndex
   private pendingReceipts?: Map<string, TxReceipt[]>
   private vmPromise?: Promise<number | null>
 
@@ -117,6 +119,11 @@ export class VMExecution extends Execution {
     }
 
     if (this.metaDB) {
+      this.txIndex = new TxIndex({
+        metaDB: this.metaDB,
+        chain: this.chain,
+        config: this.config,
+      })
       if (this.config.saveReceipts) {
         this.receiptsManager = new ReceiptsManager({
           chain: this.chain,
@@ -130,6 +137,7 @@ export class VMExecution extends Execution {
             // Once a block gets deleted from the chain, delete the receipts also
             for (const block of blocks) {
               await this.receiptsManager?.deleteReceipts(block)
+              void this.txIndex?.updateIndex(IndexOperation.Delete, IndexType.TxHash, block)
             }
             if (resolve !== undefined) {
               resolve()
@@ -319,7 +327,6 @@ export class VMExecution extends Execution {
           await this.vm.stateManager.generateCanonicalGenesis!(genesisState)
         }
       }
-
       await super.open()
       // TODO: Should a run be started to execute any left over blocks?
       // void this.run()
@@ -534,6 +541,9 @@ export class VMExecution extends Execution {
           await this.receiptsManager?.saveReceipts(block, receipts)
           this.pendingReceipts?.delete(bytesToHex(block.hash()))
         }
+        if (this.txIndex) {
+          void this.txIndex.updateIndex(IndexOperation.Save, IndexType.TxHash, block)
+        }
       }
 
       // check if the head, safe and finalized are now canonical
@@ -740,6 +750,9 @@ export class VMExecution extends Execution {
                   }
 
                   await this.receiptsManager?.saveReceipts(block, result.receipts)
+                  if (this.txIndex) {
+                    void this.txIndex.updateIndex(IndexOperation.Save, IndexType.TxHash, block)
+                  }
                   if (this.config.savePreimages && result.preimages !== undefined) {
                     await this.savePreimages(result.preimages)
                   }

packages/client/src/miner/miner.ts

@@ -13,6 +13,7 @@ import type { CliqueConfig } from '@ethereumjs/common'
 import type { Miner as EthashMiner, Solution } from '@ethereumjs/ethash'
 import type { Config } from '../config.ts'
 import type { VMExecution } from '../execution/index.ts'
+import { IndexOperation, IndexType } from '../execution/txIndex.ts'
 import type { FullEthereumService } from '../service/index.ts'
 import type { FullSynchronizer } from '../sync/index.ts'
 
@@ -336,6 +337,9 @@ export class Miner {
     if (this.config.saveReceipts) {
       await this.execution.receiptsManager?.saveReceipts(block, receipts)
     }
+    if (this.execution.txIndex) {
+      void this.execution.txIndex.updateIndex(IndexOperation.Save, IndexType.TxHash, block)
+    }
     this.config.logger?.info(
       `Miner: Sealed block with ${block.transactions.length} txs ${
         this.config.chainCommon.consensusType() === ConsensusType.ProofOfWork

packages/client/src/rpc/modules/debug.ts

@@ -176,9 +176,10 @@ export class Debug {
 
     const opts = validateTracerConfig(config)
 
-    const result = await this.service.execution.receiptsManager.getReceiptByTxHash(
-      hexToBytes(txHash),
-    )
+    if (!this.service.execution.txIndex) throw EthereumJSErrorWithoutCode('missing txIndex')
+    const txHashIndex = await this.service.execution.txIndex.getIndex(hexToBytes(txHash))
+    if (!txHashIndex) return null
+    const result = await this.service.execution.receiptsManager.getReceiptByTxHashIndex(txHashIndex)
     if (!result) return null
     const [_, blockHash, txIndex] = result
     const block = await this.service.chain.getBlock(blockHash)
@@ -445,11 +446,10 @@ export class Debug {
     const [txHash] = params
     if (!this.service.execution.receiptsManager)
       throw EthereumJSErrorWithoutCode('missing receiptsManager')
-    const result = await this.service.execution.receiptsManager.getReceiptByTxHash(
-      hexToBytes(txHash),
-    )
-    if (!result) return null
-    const [_receipt, blockHash, txIndex] = result
+    if (!this.service.execution.txIndex) throw EthereumJSErrorWithoutCode('missing txIndex')
+    const txHashIndex = await this.service.execution.txIndex.getIndex(hexToBytes(txHash))
+    if (!txHashIndex) return null
+    const [blockHash, txIndex] = txHashIndex
     const block = await this.chain.getBlock(blockHash)
     const tx = block.transactions[txIndex]
     return bytesToHex(tx.serialize())