Client: purge history helper

packages/client/src/util/inclineClient.ts

@@ -39,14 +39,14 @@
     >() as unknown as AbstractLevel<string | Uint8Array, string | Uint8Array, string | Uint8Array>
   } else {
     chainDB = new Level<string | Uint8Array, string | Uint8Array>(
-      `${datadir}/${common.chainName()}/chainDB`,
+      `${datadir}/${common.chainName()}/chain`,
     ) as unknown as AbstractLevel<string | Uint8Array, string | Uint8Array, string | Uint8Array>
 
     stateDB = new Level<string | Uint8Array, string | Uint8Array>(
-      `${datadir}/${common.chainName()}/stateDB`,
+      `${datadir}/${common.chainName()}/state`,
     ) as unknown as AbstractLevel<string | Uint8Array, string | Uint8Array, string | Uint8Array>
     metaDB = new Level<string | Uint8Array, string | Uint8Array>(
-      `${datadir}/${common.chainName()}/metaDB`,
+      `${datadir}/${common.chainName()}/meta`,
     ) as unknown as AbstractLevel<string | Uint8Array, string | Uint8Array, string | Uint8Array>
   }
   let validateConsensus = false

packages/client/src/util/index.ts

@@ -10,6 +10,7 @@ import { bytesToHex } from '@ethereumjs/util'
 export * from './inclineClient.ts'
 export * from './parse.ts'
 export * from './rpc.ts'
+export * from './purge.ts'
 // See: https://stackoverflow.com/a/50053801
 const __dirname = dirname(fileURLToPath(import.meta.url))
 

packages/client/src/util/purge.ts

@@ -0,0 +1,78 @@
+import { DBOp } from '@ethereumjs/blockchain'
+import { type BatchDBOp, type DelBatch, concatBytes, intToBytes } from '@ethereumjs/util'
+import { Level } from 'level'
+import { DBKey } from './metaDBManager.ts'
+
+export const DBTarget = {
+  NumberToHash: 4,
+  Body: 6,
+  Header: 7,
+  Receipts: 8,
+} as const
+
+async function initDBs(dataDir: string, chain: string) {
+  const chainDir = `${dataDir}/${chain}`
+
+  // Chain DB
+  const chainDataDir = `${chainDir}/chain`
+  const chainDB = new Level<string | Uint8Array, string | Uint8Array>(chainDataDir)
+  await chainDB.open()
+
+  // Meta DB (receipts, logs, indexes, skeleton chain)
+  const metaDataDir = `${chainDir}/meta`
+  const metaDB = new Level<string | Uint8Array, string | Uint8Array>(metaDataDir)
+  await metaDB.open()
+
+  return { chainDB, metaDB }
+}
+
+export async function purgeHistory(
+  dataDir: string,
+  chain: string = 'mainnet',
+  before: bigint = 15537393n,
+  headers: boolean = false,
+) {
+  const { chainDB, metaDB } = await initDBs(dataDir, chain)
+
+  const dbOps: DBOp[] = []
+  const metaDBOps: {
+    type: 'del'
+    key: Uint8Array
+  }[] = []
+  let blockNumber = before
+  while (blockNumber > 0n) {
+    const blockHashDBOp = DBOp.get(DBTarget.NumberToHash, { blockNumber })
+    const blockHash = await chainDB.get(blockHashDBOp.baseDBOp.key, {
+      keyEncoding: blockHashDBOp.baseDBOp.keyEncoding,
+      valueEncoding: blockHashDBOp.baseDBOp.valueEncoding,
+    })
+
+    if (!(blockHash instanceof Uint8Array)) {
+      blockNumber--
+      continue
+    }
+    dbOps.push(DBOp.del(DBTarget.Body, { blockHash, blockNumber }))
+    if (headers) {
+      dbOps.push(DBOp.del(DBTarget.Header, { blockHash, blockNumber }))
+    }
+    const receiptsKey = concatBytes(intToBytes(DBKey.Receipts), blockHash)
+    metaDBOps.push({
+      type: 'del',
+      key: receiptsKey,
+    })
+    blockNumber--
+  }
+  const convertedOps: BatchDBOp[] = dbOps.map((op) => {
+    const convertedOp = {
+      key: op.baseDBOp.key,
+      type: 'del',
+      opts: {
+        keyEncoding: op.baseDBOp.keyEncoding,
+      },
+    }
+    return convertedOp as DelBatch
+  })
+
+  await chainDB.batch(convertedOps)
+  await metaDB.batch(metaDBOps, { keyEncoding: 'view' })
+}

packages/client/test/integration/purge.spec.ts

@@ -0,0 +1,123 @@
+import { rmSync } from 'fs'
+import { resolve } from 'path'
+import { Hardfork, createCommonFromGethGenesis } from '@ethereumjs/common'
+import {
+  Address,
+  bytesToHex,
+  concatBytes,
+  hexToBytes,
+  parseGethGenesisState,
+} from '@ethereumjs/util'
+import { assert, afterAll, beforeAll, describe, it } from 'vitest'
+import type { EthereumClient } from '../../src/client.ts'
+import { Config } from '../../src/config.ts'
+import { getLogger } from '../../src/logging.ts'
+import { Event } from '../../src/types.ts'
+import { createInlineClient, purgeHistory } from '../../src/util/index.ts'
+
+async function setupDevnet(prefundAddress: Address) {
+  const addr = prefundAddress.toString().slice(2)
+  const consensusConfig = {
+    clique: {
+      period: 1,
+      epoch: 30000,
+    },
+  }
+  const defaultChainData = {
+    config: {
+      chainId: 123456,
+      homesteadBlock: 0,
+      eip150Block: 0,
+      eip150Hash: '0x0000000000000000000000000000000000000000000000000000000000000000',
+      eip155Block: 0,
+      eip158Block: 0,
+      byzantiumBlock: 0,
+      constantinopleBlock: 0,
+      petersburgBlock: 0,
+      istanbulBlock: 0,
+      berlinBlock: 0,
+      londonBlock: 0,
+      ...consensusConfig,
+    },
+    nonce: '0x0',
+    timestamp: '0x614b3731',
+    gasLimit: '0x47b760',
+    difficulty: '0x1',
+    mixHash: '0x0000000000000000000000000000000000000000000000000000000000000000',
+    coinbase: '0x0000000000000000000000000000000000000000',
+    number: '0x0',
+    gasUsed: '0x0',
+    parentHash: '0x0000000000000000000000000000000000000000000000000000000000000000',
+    baseFeePerGas: 7,
+  }
+  const extraData = concatBytes(new Uint8Array(32), prefundAddress.toBytes(), new Uint8Array(65))
+
+  const chainData = {
+    ...defaultChainData,
+    extraData: bytesToHex(extraData),
+    alloc: { [addr]: { balance: '0x10000000000000000000' } },
+  }
+
+  const common = createCommonFromGethGenesis(chainData, {
+    chain: 'devnet',
+    hardfork: Hardfork.London,
+  })
+  const customGenesisState = parseGethGenesisState(chainData)
+  return { common, customGenesisState }
+}
+
+const accounts: [Address, Uint8Array][] = [
+  [
+    new Address(hexToBytes('0x0b90087d864e82a284dca15923f3776de6bb016f')),
+    hexToBytes('0x64bf9cc30328b0e42387b3c82c614e6386259136235e20c1357bd11cdee86993'),
+  ],
+]
+
+async function minerSetup(): Promise<EthereumClient[]> {
+  const { common, customGenesisState } = await setupDevnet(accounts[0][0])
+  const config1 = new Config({
+    common,
+    accountCache: 10000,
+    storageCache: 1000,
+    mine: true,
+    accounts,
+    logger: getLogger({ logLevel: 'warn' }),
+  })
+
+  const miner = await createInlineClient(config1, common, customGenesisState, './datadir', false)
+
+  return [miner]
+}
+
+describe('should mine blocks and then purge a few', () => {
+  beforeAll(() => {
+    rmSync(resolve(__dirname, '../../datadir/devnet'), { recursive: true, force: true })
+  })
+  it('should work', async () => {
+    const [miner] = await minerSetup()
+
+    const targetHeight = BigInt(5)
+    await new Promise((resolve) => {
+      miner.config.events.on(Event.SYNC_SYNCHRONIZED, (chainHeight) => {
+        if (chainHeight === targetHeight) {
+          assert.equal(miner.chain.blocks.height, targetHeight, 'synced blocks successfully')
+          resolve(undefined)
+        }
+      })
+    })
+
+    await miner.stop()
+    // We have to manually close the dbs or we'll get a db lock error
+    // @ts-expect-error leveldb is not visible in interface (but it's there!)
+    await miner.chain.chainDB['_leveldb'].close()
+    await miner.service.execution['metaDB']?.close()
+
+    // Purge history prior to block 3 and delete headers
+    await purgeHistory('./datadir', 'devnet', 2n, true)
+
+    assert.throws(async () => miner.chain.getBlock(1n), 'should not have block 1')
+  }, 60000)
+  afterAll(() => {
+    rmSync(resolve(__dirname, '../../datadir/devnet'), { recursive: true, force: true })
+  })
+})