From d0f03c288d35c428717da542e782873df8b5671e Mon Sep 17 00:00:00 2001 From: ScottyPoi Date: Mon, 24 Mar 2025 17:54:01 -0600 Subject: [PATCH 1/4] client: add helper function to purge history from DB --- packages/client/src/util/purge.ts | 67 +++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 packages/client/src/util/purge.ts diff --git a/packages/client/src/util/purge.ts b/packages/client/src/util/purge.ts new file mode 100644 index 00000000000..4567ac7a223 --- /dev/null +++ b/packages/client/src/util/purge.ts @@ -0,0 +1,67 @@ +import { DBOp } from '@ethereumjs/blockchain' +import { type BatchDBOp, type DelBatch, concatBytes, intToBytes } from '@ethereumjs/util' +import { Level } from 'level' +import { DBTarget } from '../../../blockchain/dist/esm/db/operation.js' +import { DBKey } from './metaDBManager.ts' + +async function initDBs(dataDir: string, chain: string) { + const chainDir = `${dataDir}/${chain}` + // Chain DB + const chainDataDir = `${chainDir}/chain` + const chainDB = new Level(chainDataDir) + await chainDB.open() + // Meta DB (receipts, logs, indexes, skeleton chain) + const metaDataDir = `${chainDir}/meta` + const metaDB = new Level(metaDataDir) + await metaDB.open() + + return { chainDB, metaDB } +} + +export async function purgeHistory( + dataDir: string, + chain: string = 'mainnet', + before: bigint = 15537393n, + headers: boolean = false, +) { + const { chainDB, metaDB } = await initDBs(dataDir, chain) + const dbOps: DBOp[] = [] + const metaDBOps: { + type: 'del' + key: Uint8Array + }[] = [] + let blockNumber = before + while (blockNumber > 0n) { + const blockHashDBOp = DBOp.get(DBTarget.NumberToHash, { blockNumber }) + const blockHash = await chainDB.get(blockHashDBOp.baseDBOp.key, { + keyEncoding: blockHashDBOp.baseDBOp.keyEncoding, + valueEncoding: blockHashDBOp.baseDBOp.valueEncoding, + }) + if (!(blockHash instanceof Uint8Array)) { + blockNumber-- + continue + } + dbOps.push(DBOp.del(DBTarget.Body, { blockHash, blockNumber })) + if (headers) { + dbOps.push(DBOp.del(DBTarget.Header, { blockHash, blockNumber })) + } + const receiptsKey = concatBytes(intToBytes(DBKey.Receipts), blockHash) + metaDBOps.push({ + type: 'del', + key: receiptsKey, + }) + blockNumber-- + } + const convertedOps: BatchDBOp[] = dbOps.map((op) => { + const convertedOp = { + key: op.baseDBOp.key, + type: 'del', + opts: { + keyEncoding: op.baseDBOp.keyEncoding, + }, + } + return convertedOp as DelBatch + }) + await chainDB.batch(convertedOps) + await metaDB.batch(metaDBOps, { keyEncoding: 'view' }) +} From d41f56aa4dc0337caf7be0042b2ff2656b5f7d26 Mon Sep 17 00:00:00 2001 From: ScottyPoi Date: Mon, 24 Mar 2025 17:56:14 -0600 Subject: [PATCH 2/4] export from index --- packages/client/src/util/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/client/src/util/index.ts b/packages/client/src/util/index.ts index 4988f9074e1..aea45fb8e4d 100644 --- a/packages/client/src/util/index.ts +++ b/packages/client/src/util/index.ts @@ -10,6 +10,7 @@ import { bytesToHex } from '@ethereumjs/util' export * from './inclineClient.ts' export * from './parse.ts' export * from './rpc.ts' +export * from './purge.ts' // See: https://stackoverflow.com/a/50053801 const __dirname = dirname(fileURLToPath(import.meta.url)) From fec3669ed4cbff526922170a711446c9196c2a2d Mon Sep 17 00:00:00 2001 From: ScottyPoi Date: Mon, 24 Mar 2025 18:06:35 -0600 Subject: [PATCH 3/4] remove dist export --- packages/client/src/util/purge.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/client/src/util/purge.ts b/packages/client/src/util/purge.ts index 4567ac7a223..7a90378890d 100644 --- a/packages/client/src/util/purge.ts +++ b/packages/client/src/util/purge.ts @@ -1,9 +1,15 @@ import { DBOp } from '@ethereumjs/blockchain' import { type BatchDBOp, type DelBatch, concatBytes, intToBytes } from '@ethereumjs/util' import { Level } from 'level' -import { DBTarget } from '../../../blockchain/dist/esm/db/operation.js' import { DBKey } from './metaDBManager.ts' +export const DBTarget = { + NumberToHash: 4, + Body: 6, + Header: 7, + Receipts: 8, +} as const + async function initDBs(dataDir: string, chain: string) { const chainDir = `${dataDir}/${chain}` // Chain DB From eddc0abbd40a09fc782d3597fb4fce9c172d5a45 Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Fri, 28 Mar 2025 17:32:49 -0400 Subject: [PATCH 4/4] Add test --- packages/client/src/util/inclineClient.ts | 6 +- packages/client/src/util/purge.ts | 5 + .../client/test/integration/purge.spec.ts | 123 ++++++++++++++++++ 3 files changed, 131 insertions(+), 3 deletions(-) create mode 100644 packages/client/test/integration/purge.spec.ts diff --git a/packages/client/src/util/inclineClient.ts b/packages/client/src/util/inclineClient.ts index c2d295f471d..be5b05ebf82 100644 --- a/packages/client/src/util/inclineClient.ts +++ b/packages/client/src/util/inclineClient.ts @@ -39,14 +39,14 @@ export async function createInlineClient( >() as unknown as AbstractLevel } else { chainDB = new Level( - `${datadir}/${common.chainName()}/chainDB`, + `${datadir}/${common.chainName()}/chain`, ) as unknown as AbstractLevel stateDB = new Level( - `${datadir}/${common.chainName()}/stateDB`, + `${datadir}/${common.chainName()}/state`, ) as unknown as AbstractLevel metaDB = new Level( - `${datadir}/${common.chainName()}/metaDB`, + `${datadir}/${common.chainName()}/meta`, ) as unknown as AbstractLevel } let validateConsensus = false diff --git a/packages/client/src/util/purge.ts b/packages/client/src/util/purge.ts index 7a90378890d..afc2705fd6f 100644 --- a/packages/client/src/util/purge.ts +++ b/packages/client/src/util/purge.ts @@ -12,10 +12,12 @@ export const DBTarget = { async function initDBs(dataDir: string, chain: string) { const chainDir = `${dataDir}/${chain}` + // Chain DB const chainDataDir = `${chainDir}/chain` const chainDB = new Level(chainDataDir) await chainDB.open() + // Meta DB (receipts, logs, indexes, skeleton chain) const metaDataDir = `${chainDir}/meta` const metaDB = new Level(metaDataDir) @@ -31,6 +33,7 @@ export async function purgeHistory( headers: boolean = false, ) { const { chainDB, metaDB } = await initDBs(dataDir, chain) + const dbOps: DBOp[] = [] const metaDBOps: { type: 'del' @@ -43,6 +46,7 @@ export async function purgeHistory( keyEncoding: blockHashDBOp.baseDBOp.keyEncoding, valueEncoding: blockHashDBOp.baseDBOp.valueEncoding, }) + if (!(blockHash instanceof Uint8Array)) { blockNumber-- continue @@ -68,6 +72,7 @@ export async function purgeHistory( } return convertedOp as DelBatch }) + await chainDB.batch(convertedOps) await metaDB.batch(metaDBOps, { keyEncoding: 'view' }) } diff --git a/packages/client/test/integration/purge.spec.ts b/packages/client/test/integration/purge.spec.ts new file mode 100644 index 00000000000..bc9263a794a --- /dev/null +++ b/packages/client/test/integration/purge.spec.ts @@ -0,0 +1,123 @@ +import { rmSync } from 'fs' +import { resolve } from 'path' +import { Hardfork, createCommonFromGethGenesis } from '@ethereumjs/common' +import { + Address, + bytesToHex, + concatBytes, + hexToBytes, + parseGethGenesisState, +} from '@ethereumjs/util' +import { assert, afterAll, beforeAll, describe, it } from 'vitest' +import type { EthereumClient } from '../../src/client.ts' +import { Config } from '../../src/config.ts' +import { getLogger } from '../../src/logging.ts' +import { Event } from '../../src/types.ts' +import { createInlineClient, purgeHistory } from '../../src/util/index.ts' + +async function setupDevnet(prefundAddress: Address) { + const addr = prefundAddress.toString().slice(2) + const consensusConfig = { + clique: { + period: 1, + epoch: 30000, + }, + } + const defaultChainData = { + config: { + chainId: 123456, + homesteadBlock: 0, + eip150Block: 0, + eip150Hash: '0x0000000000000000000000000000000000000000000000000000000000000000', + eip155Block: 0, + eip158Block: 0, + byzantiumBlock: 0, + constantinopleBlock: 0, + petersburgBlock: 0, + istanbulBlock: 0, + berlinBlock: 0, + londonBlock: 0, + ...consensusConfig, + }, + nonce: '0x0', + timestamp: '0x614b3731', + gasLimit: '0x47b760', + difficulty: '0x1', + mixHash: '0x0000000000000000000000000000000000000000000000000000000000000000', + coinbase: '0x0000000000000000000000000000000000000000', + number: '0x0', + gasUsed: '0x0', + parentHash: '0x0000000000000000000000000000000000000000000000000000000000000000', + baseFeePerGas: 7, + } + const extraData = concatBytes(new Uint8Array(32), prefundAddress.toBytes(), new Uint8Array(65)) + + const chainData = { + ...defaultChainData, + extraData: bytesToHex(extraData), + alloc: { [addr]: { balance: '0x10000000000000000000' } }, + } + + const common = createCommonFromGethGenesis(chainData, { + chain: 'devnet', + hardfork: Hardfork.London, + }) + const customGenesisState = parseGethGenesisState(chainData) + return { common, customGenesisState } +} + +const accounts: [Address, Uint8Array][] = [ + [ + new Address(hexToBytes('0x0b90087d864e82a284dca15923f3776de6bb016f')), + hexToBytes('0x64bf9cc30328b0e42387b3c82c614e6386259136235e20c1357bd11cdee86993'), + ], +] + +async function minerSetup(): Promise { + const { common, customGenesisState } = await setupDevnet(accounts[0][0]) + const config1 = new Config({ + common, + accountCache: 10000, + storageCache: 1000, + mine: true, + accounts, + logger: getLogger({ logLevel: 'warn' }), + }) + + const miner = await createInlineClient(config1, common, customGenesisState, './datadir', false) + + return [miner] +} + +describe('should mine blocks and then purge a few', () => { + beforeAll(() => { + rmSync(resolve(__dirname, '../../datadir/devnet'), { recursive: true, force: true }) + }) + it('should work', async () => { + const [miner] = await minerSetup() + + const targetHeight = BigInt(5) + await new Promise((resolve) => { + miner.config.events.on(Event.SYNC_SYNCHRONIZED, (chainHeight) => { + if (chainHeight === targetHeight) { + assert.equal(miner.chain.blocks.height, targetHeight, 'synced blocks successfully') + resolve(undefined) + } + }) + }) + + await miner.stop() + // We have to manually close the dbs or we'll get a db lock error + // @ts-expect-error leveldb is not visible in interface (but it's there!) + await miner.chain.chainDB['_leveldb'].close() + await miner.service.execution['metaDB']?.close() + + // Purge history prior to block 3 and delete headers + await purgeHistory('./datadir', 'devnet', 2n, true) + + assert.throws(async () => miner.chain.getBlock(1n), 'should not have block 1') + }, 60000) + afterAll(() => { + rmSync(resolve(__dirname, '../../datadir/devnet'), { recursive: true, force: true }) + }) +})