core/rawdb, triedb/pathdb: implement history indexer (#31156)

This pull request is part-1 for shipping the core part of archive node
in PBSS mode.

Author: rjl493456442

cmd/utils/flags.go

@@ -1661,11 +1661,6 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 			cfg.TransactionHistory = 0
 			log.Warn("Disabled transaction unindexing for archive node")
 		}
-
-		if cfg.StateScheme != rawdb.HashScheme {
-			cfg.StateScheme = rawdb.HashScheme
-			log.Warn("Forcing hash state-scheme for archive mode")
-		}
 	}
 	if ctx.IsSet(LogHistoryFlag.Name) {
 		cfg.LogHistory = ctx.Uint64(LogHistoryFlag.Name)

core/blockchain.go

@@ -242,9 +242,10 @@ func (cfg *BlockChainConfig) triedbConfig(isVerkle bool) *triedb.Config {
 	}
 	if cfg.StateScheme == rawdb.PathScheme {
 		config.PathDB = &pathdb.Config{
-			StateHistory:   cfg.StateHistory,
-			TrieCleanSize:  cfg.TrieCleanLimit * 1024 * 1024,
-			StateCleanSize: cfg.SnapshotLimit * 1024 * 1024,
+			StateHistory:        cfg.StateHistory,
+			EnableStateIndexing: cfg.ArchiveMode,
+			TrieCleanSize:       cfg.TrieCleanLimit * 1024 * 1024,
+			StateCleanSize:      cfg.SnapshotLimit * 1024 * 1024,
 
 			// TODO(rjl493456442): The write buffer represents the memory limit used
 			// for flushing both trie data and state data to disk. The config name

core/rawdb/accessors_history.go

@@ -0,0 +1,179 @@
+// Copyright 2025 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package rawdb
+
+import (
+	"bytes"
+	"errors"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// ReadStateHistoryIndexMetadata retrieves the metadata of state history index.
+func ReadStateHistoryIndexMetadata(db ethdb.KeyValueReader) []byte {
+	data, _ := db.Get(headStateHistoryIndexKey)
+	return data
+}
+
+// WriteStateHistoryIndexMetadata stores the metadata of state history index
+// into database.
+func WriteStateHistoryIndexMetadata(db ethdb.KeyValueWriter, blob []byte) {
+	if err := db.Put(headStateHistoryIndexKey, blob); err != nil {
+		log.Crit("Failed to store the metadata of state history index", "err", err)
+	}
+}
+
+// DeleteStateHistoryIndexMetadata removes the metadata of state history index.
+func DeleteStateHistoryIndexMetadata(db ethdb.KeyValueWriter) {
+	if err := db.Delete(headStateHistoryIndexKey); err != nil {
+		log.Crit("Failed to delete the metadata of state history index", "err", err)
+	}
+}
+
+// ReadAccountHistoryIndex retrieves the account history index with the provided
+// account address.
+func ReadAccountHistoryIndex(db ethdb.KeyValueReader, addressHash common.Hash) []byte {
+	data, err := db.Get(accountHistoryIndexKey(addressHash))
+	if err != nil || len(data) == 0 {
+		return nil
+	}
+	return data
+}
+
+// WriteAccountHistoryIndex writes the provided account history index into database.
+func WriteAccountHistoryIndex(db ethdb.KeyValueWriter, addressHash common.Hash, data []byte) {
+	if err := db.Put(accountHistoryIndexKey(addressHash), data); err != nil {
+		log.Crit("Failed to store account history index", "err", err)
+	}
+}
+
+// DeleteAccountHistoryIndex deletes the specified account history index from
+// the database.
+func DeleteAccountHistoryIndex(db ethdb.KeyValueWriter, addressHash common.Hash) {
+	if err := db.Delete(accountHistoryIndexKey(addressHash)); err != nil {
+		log.Crit("Failed to delete account history index", "err", err)
+	}
+}
+
+// ReadStorageHistoryIndex retrieves the storage history index with the provided
+// account address and storage key hash.
+func ReadStorageHistoryIndex(db ethdb.KeyValueReader, addressHash common.Hash, storageHash common.Hash) []byte {
+	data, err := db.Get(storageHistoryIndexKey(addressHash, storageHash))
+	if err != nil || len(data) == 0 {
+		return nil
+	}
+	return data
+}
+
+// WriteStorageHistoryIndex writes the provided storage history index into database.
+func WriteStorageHistoryIndex(db ethdb.KeyValueWriter, addressHash common.Hash, storageHash common.Hash, data []byte) {
+	if err := db.Put(storageHistoryIndexKey(addressHash, storageHash), data); err != nil {
+		log.Crit("Failed to store storage history index", "err", err)
+	}
+}
+
+// DeleteStorageHistoryIndex deletes the specified state index from the database.
+func DeleteStorageHistoryIndex(db ethdb.KeyValueWriter, addressHash common.Hash, storageHash common.Hash) {
+	if err := db.Delete(storageHistoryIndexKey(addressHash, storageHash)); err != nil {
+		log.Crit("Failed to delete storage history index", "err", err)
+	}
+}
+
+// ReadAccountHistoryIndexBlock retrieves the index block with the provided
+// account address along with the block id.
+func ReadAccountHistoryIndexBlock(db ethdb.KeyValueReader, addressHash common.Hash, blockID uint32) []byte {
+	data, err := db.Get(accountHistoryIndexBlockKey(addressHash, blockID))
+	if err != nil || len(data) == 0 {
+		return nil
+	}
+	return data
+}
+
+// WriteAccountHistoryIndexBlock writes the provided index block into database.
+func WriteAccountHistoryIndexBlock(db ethdb.KeyValueWriter, addressHash common.Hash, blockID uint32, data []byte) {
+	if err := db.Put(accountHistoryIndexBlockKey(addressHash, blockID), data); err != nil {
+		log.Crit("Failed to store account index block", "err", err)
+	}
+}
+
+// DeleteAccountHistoryIndexBlock deletes the specified index block from the database.
+func DeleteAccountHistoryIndexBlock(db ethdb.KeyValueWriter, addressHash common.Hash, blockID uint32) {
+	if err := db.Delete(accountHistoryIndexBlockKey(addressHash, blockID)); err != nil {
+		log.Crit("Failed to delete account index block", "err", err)
+	}
+}
+
+// ReadStorageHistoryIndexBlock retrieves the index block with the provided state
+// identifier along with the block id.
+func ReadStorageHistoryIndexBlock(db ethdb.KeyValueReader, addressHash common.Hash, storageHash common.Hash, blockID uint32) []byte {
+	data, err := db.Get(storageHistoryIndexBlockKey(addressHash, storageHash, blockID))
+	if err != nil || len(data) == 0 {
+		return nil
+	}
+	return data
+}
+
+// WriteStorageHistoryIndexBlock writes the provided index block into database.
+func WriteStorageHistoryIndexBlock(db ethdb.KeyValueWriter, addressHash common.Hash, storageHash common.Hash, id uint32, data []byte) {
+	if err := db.Put(storageHistoryIndexBlockKey(addressHash, storageHash, id), data); err != nil {
+		log.Crit("Failed to store storage index block", "err", err)
+	}
+}
+
+// DeleteStorageHistoryIndexBlock deletes the specified index block from the database.
+func DeleteStorageHistoryIndexBlock(db ethdb.KeyValueWriter, addressHash common.Hash, storageHash common.Hash, id uint32) {
+	if err := db.Delete(storageHistoryIndexBlockKey(addressHash, storageHash, id)); err != nil {
+		log.Crit("Failed to delete storage index block", "err", err)
+	}
+}
+
+// increaseKey increase the input key by one bit. Return nil if the entire
+// addition operation overflows.
+func increaseKey(key []byte) []byte {
+	for i := len(key) - 1; i >= 0; i-- {
+		key[i]++
+		if key[i] != 0x0 {
+			return key
+		}
+	}
+	return nil
+}
+
+// DeleteStateHistoryIndex completely removes all history indexing data, including
+// indexes for accounts and storages.
+//
+// Note, this method assumes the storage space with prefix `StateHistoryIndexPrefix`
+// is exclusively occupied by the history indexing data!
+func DeleteStateHistoryIndex(db ethdb.KeyValueRangeDeleter) {
+	start := StateHistoryIndexPrefix
+	limit := increaseKey(bytes.Clone(StateHistoryIndexPrefix))
+
+	// Try to remove the data in the range by a loop, as the leveldb
+	// doesn't support the native range deletion.
+	for {
+		err := db.DeleteRange(start, limit)
+		if err == nil {
+			return
+		}
+		if errors.Is(err, ethdb.ErrTooManyKeys) {
+			continue
+		}
+		log.Crit("Failed to delete history index range", "err", err)
+	}
+}

core/rawdb/accessors_state.go

@@ -18,6 +18,7 @@ package rawdb
 
 import (
 	"encoding/binary"
+	"errors"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/ethdb"
@@ -255,6 +256,36 @@ func ReadStateHistory(db ethdb.AncientReaderOp, id uint64) ([]byte, []byte, []by
 	return meta, accountIndex, storageIndex, accountData, storageData, nil
 }
 
+// ReadStateHistoryList retrieves a list of state histories from database with
+// specific range. Compute the position of state history in freezer by minus one
+// since the id of first state history starts from one(zero for initial state).
+func ReadStateHistoryList(db ethdb.AncientReaderOp, start uint64, count uint64) ([][]byte, [][]byte, [][]byte, [][]byte, [][]byte, error) {
+	metaList, err := db.AncientRange(stateHistoryMeta, start-1, count, 0)
+	if err != nil {
+		return nil, nil, nil, nil, nil, err
+	}
+	aIndexList, err := db.AncientRange(stateHistoryAccountIndex, start-1, count, 0)
+	if err != nil {
+		return nil, nil, nil, nil, nil, err
+	}
+	sIndexList, err := db.AncientRange(stateHistoryStorageIndex, start-1, count, 0)
+	if err != nil {
+		return nil, nil, nil, nil, nil, err
+	}
+	aDataList, err := db.AncientRange(stateHistoryAccountData, start-1, count, 0)
+	if err != nil {
+		return nil, nil, nil, nil, nil, err
+	}
+	sDataList, err := db.AncientRange(stateHistoryStorageData, start-1, count, 0)
+	if err != nil {
+		return nil, nil, nil, nil, nil, err
+	}
+	if len(metaList) != len(aIndexList) || len(metaList) != len(sIndexList) || len(metaList) != len(aDataList) || len(metaList) != len(sDataList) {
+		return nil, nil, nil, nil, nil, errors.New("state history is corrupted")
+	}
+	return metaList, aIndexList, sIndexList, aDataList, sDataList, nil
+}
+
 // WriteStateHistory writes the provided state history to database. Compute the
 // position of state history in freezer by minus one since the id of first state
 // history starts from one(zero for initial state).

core/rawdb/database.go

@@ -412,6 +412,9 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 		filterMapLastBlock stat
 		filterMapBlockLV   stat
 
+		// Path-mode archive data
+		stateIndex stat
+
 		// Verkle statistics
 		verkleTries        stat
 		verkleStateLookups stat
@@ -489,6 +492,10 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 		case bytes.HasPrefix(key, bloomBitsMetaPrefix) && len(key) < len(bloomBitsMetaPrefix)+8:
 			bloomBits.Add(size)
 
+		// Path-based historic state indexes
+		case bytes.HasPrefix(key, StateHistoryIndexPrefix) && len(key) >= len(StateHistoryIndexPrefix)+common.HashLength:
+			stateIndex.Add(size)
+
 		// Verkle trie data is detected, determine the sub-category
 		case bytes.HasPrefix(key, VerklePrefix):
 			remain := key[len(VerklePrefix):]
@@ -544,6 +551,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 		{"Key-Value store", "Path trie state lookups", stateLookups.Size(), stateLookups.Count()},
 		{"Key-Value store", "Path trie account nodes", accountTries.Size(), accountTries.Count()},
 		{"Key-Value store", "Path trie storage nodes", storageTries.Size(), storageTries.Count()},
+		{"Key-Value store", "Path state history indexes", stateIndex.Size(), stateIndex.Count()},
 		{"Key-Value store", "Verkle trie nodes", verkleTries.Size(), verkleTries.Count()},
 		{"Key-Value store", "Verkle trie state lookups", verkleStateLookups.Size(), verkleStateLookups.Count()},
 		{"Key-Value store", "Trie preimages", preimages.Size(), preimages.Count()},
@@ -591,7 +599,7 @@ var knownMetadataKeys = [][]byte{
 	snapshotGeneratorKey, snapshotRecoveryKey, txIndexTailKey, fastTxLookupLimitKey,
 	uncleanShutdownKey, badBlockKey, transitionStatusKey, skeletonSyncStatusKey,
 	persistentStateIDKey, trieJournalKey, snapshotSyncStatusKey, snapSyncStatusFlagKey,
-	filterMapsRangeKey,
+	filterMapsRangeKey, headStateHistoryIndexKey,
 }
 
 // printChainMetadata prints out chain metadata to stderr.

core/rawdb/schema.go

@@ -76,6 +76,10 @@ var (
 	// trieJournalKey tracks the in-memory trie node layers across restarts.
 	trieJournalKey = []byte("TrieJournal")
 
+	// headStateHistoryIndexKey tracks the ID of the latest state history that has
+	// been indexed.
+	headStateHistoryIndexKey = []byte("LastStateHistoryIndex")
+
 	// txIndexTailKey tracks the oldest block whose transactions have been indexed.
 	txIndexTailKey = []byte("TransactionIndexTail")
 
@@ -117,6 +121,13 @@ var (
 	TrieNodeStoragePrefix = []byte("O") // TrieNodeStoragePrefix + accountHash + hexPath -> trie node
 	stateIDPrefix         = []byte("L") // stateIDPrefix + state root -> state id
 
+	// State history indexing within path-based storage scheme
+	StateHistoryIndexPrefix           = []byte("m")   // The global prefix of state history index data
+	StateHistoryAccountMetadataPrefix = []byte("ma")  // StateHistoryAccountMetadataPrefix + account address hash => account metadata
+	StateHistoryStorageMetadataPrefix = []byte("ms")  // StateHistoryStorageMetadataPrefix + account address hash + storage slot hash => slot metadata
+	StateHistoryAccountBlockPrefix    = []byte("mba") // StateHistoryAccountBlockPrefix + account address hash + block_number => account block
+	StateHistoryStorageBlockPrefix    = []byte("mbs") // StateHistoryStorageBlockPrefix + account address hash + storage slot hash + block_number => slot block
+
 	// VerklePrefix is the database prefix for Verkle trie data, which includes:
 	// (a) Trie nodes
 	// (b) In-memory trie node journal
@@ -362,3 +373,27 @@ func filterMapBlockLVKey(number uint64) []byte {
 	binary.BigEndian.PutUint64(key[l:], number)
 	return key
 }
+
+// accountHistoryIndexKey = StateHistoryAccountMetadataPrefix + addressHash
+func accountHistoryIndexKey(addressHash common.Hash) []byte {
+	return append(StateHistoryAccountMetadataPrefix, addressHash.Bytes()...)
+}
+
+// storageHistoryIndexKey = StateHistoryStorageMetadataPrefix + addressHash + storageHash
+func storageHistoryIndexKey(addressHash common.Hash, storageHash common.Hash) []byte {
+	return append(append(StateHistoryStorageMetadataPrefix, addressHash.Bytes()...), storageHash.Bytes()...)
+}
+
+// accountHistoryIndexBlockKey = StateHistoryAccountBlockPrefix + addressHash + blockID
+func accountHistoryIndexBlockKey(addressHash common.Hash, blockID uint32) []byte {
+	var buf [4]byte
+	binary.BigEndian.PutUint32(buf[:], blockID)
+	return append(append(StateHistoryAccountBlockPrefix, addressHash.Bytes()...), buf[:]...)
+}
+
+// storageHistoryIndexBlockKey = StateHistoryStorageBlockPrefix + addressHash + storageHash + blockID
+func storageHistoryIndexBlockKey(addressHash common.Hash, storageHash common.Hash, blockID uint32) []byte {
+	var buf [4]byte
+	binary.BigEndian.PutUint32(buf[:], blockID)
+	return append(append(append(StateHistoryStorageBlockPrefix, addressHash.Bytes()...), storageHash.Bytes()...), buf[:]...)
+}

ethdb/pebble/pebble.go

@@ -83,6 +83,7 @@ type Database struct {
 	estimatedCompDebtGauge *metrics.Gauge   // Gauge for tracking the number of bytes that need to be compacted
 	liveCompGauge          *metrics.Gauge   // Gauge for tracking the number of in-progress compactions
 	liveCompSizeGauge      *metrics.Gauge   // Gauge for tracking the size of in-progress compactions
+	liveIterGauge          *metrics.Gauge   // Gauge for tracking the number of live database iterators
 	levelsGauge            []*metrics.Gauge // Gauge for tracking the number of tables in levels
 
 	quitLock sync.RWMutex    // Mutex protecting the quit channel and the closed flag
@@ -326,6 +327,7 @@ func New(file string, cache int, handles int, namespace string, readonly bool) (
 	db.estimatedCompDebtGauge = metrics.GetOrRegisterGauge(namespace+"compact/estimateDebt", nil)
 	db.liveCompGauge = metrics.GetOrRegisterGauge(namespace+"compact/live/count", nil)
 	db.liveCompSizeGauge = metrics.GetOrRegisterGauge(namespace+"compact/live/size", nil)
+	db.liveIterGauge = metrics.GetOrRegisterGauge(namespace+"iter/count", nil)
 
 	// Start up the metrics gathering and return
 	go db.meter(metricsGatheringInterval, namespace)
@@ -582,6 +584,7 @@ func (d *Database) meter(refresh time.Duration, namespace string) {
 		d.seekCompGauge.Update(stats.Compact.ReadCount)
 		d.liveCompGauge.Update(stats.Compact.NumInProgress)
 		d.liveCompSizeGauge.Update(stats.Compact.InProgressBytes)
+		d.liveIterGauge.Update(stats.TableIters)
 
 		d.liveMemTablesGauge.Update(stats.MemTable.Count)
 		d.zombieMemTablesGauge.Update(stats.MemTable.ZombieCount)