triedb/pathdb, core: keep root->id mappings after truncation (#32502)

This pull request preserves the root->ID mappings in the path database
even after the associated state histories are truncated, regardless of
whether the truncation occurs at the head or the tail.

The motivation is to support an additional history type, trienode history. 
Since the root->ID mappings are shared between two history instances, 
they must not be removed by either one.

As a consequence, the root->ID mappings remain in the database even
after the corresponding histories are pruned. While these mappings may 
become  dangling, it is safe and cheap to keep them.

Additionally, this pull request enhances validation during historical
reader construction, ensuring that only canonical historical state will be
served.

Author: rjl493456442

core/rawdb/accessors_state.go

@@ -119,13 +119,6 @@ func WriteStateID(db ethdb.KeyValueWriter, root common.Hash, id uint64) {
 	}
 }
 
-// DeleteStateID deletes the specified state lookup from the database.
-func DeleteStateID(db ethdb.KeyValueWriter, root common.Hash) {
-	if err := db.Delete(stateIDKey(root)); err != nil {
-		log.Crit("Failed to delete state ID", "err", err)
-	}
-}
-
 // ReadPersistentStateID retrieves the id of the persistent state from the database.
 func ReadPersistentStateID(db ethdb.KeyValueReader) uint64 {
 	data, _ := db.Get(persistentStateIDKey)

triedb/pathdb/database.go

@@ -334,7 +334,7 @@ func (db *Database) repairHistory() error {
 	}
 	// Truncate the extra state histories above in freezer in case it's not
 	// aligned with the disk layer. It might happen after a unclean shutdown.
-	pruned, err := truncateFromHead(db.diskdb, db.stateFreezer, id)
+	pruned, err := truncateFromHead(db.stateFreezer, id)
 	if err != nil {
 		log.Crit("Failed to truncate extra state histories", "err", err)
 	}
@@ -590,7 +590,7 @@ func (db *Database) Recover(root common.Hash) error {
 	if err := db.diskdb.SyncKeyValue(); err != nil {
 		return err
 	}
-	_, err := truncateFromHead(db.diskdb, db.stateFreezer, dl.stateID())
+	_, err := truncateFromHead(db.stateFreezer, dl.stateID())
 	if err != nil {
 		return err
 	}
@@ -615,14 +615,14 @@ func (db *Database) Recoverable(root common.Hash) bool {
 		return false
 	}
 	// This is a temporary workaround for the unavailability of the freezer in
-	// dev mode. As a consequence, the Pathdb loses the ability for deep reorg
+	// dev mode. As a consequence, the database loses the ability for deep reorg
 	// in certain cases.
 	// TODO(rjl493456442): Implement the in-memory ancient store.
 	if db.stateFreezer == nil {
 		return false
 	}
 	// Ensure the requested state is a canonical state and all state
-	// histories in range [id+1, disklayer.ID] are present and complete.
+	// histories in range [id+1, dl.ID] are present and complete.
 	return checkStateHistories(db.stateFreezer, *id+1, dl.stateID()-*id, func(m *meta) error {
 		if m.parent != root {
 			return errors.New("unexpected state history")

triedb/pathdb/disklayer.go

@@ -378,7 +378,7 @@ func (dl *diskLayer) writeStateHistory(diff *diffLayer) (bool, error) {
 		log.Debug("Skip tail truncation", "persistentID", persistentID, "tailID", tail+1, "headID", diff.stateID(), "limit", limit)
 		return true, nil
 	}
-	pruned, err := truncateFromTail(dl.db.diskdb, dl.db.stateFreezer, newFirst-1)
+	pruned, err := truncateFromTail(dl.db.stateFreezer, newFirst-1)
 	if err != nil {
 		return false, err
 	}

triedb/pathdb/history.go

@@ -0,0 +1,87 @@
+// Copyright 2025 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/
+
+package pathdb
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+var (
+	errHeadTruncationOutOfRange = errors.New("history head truncation out of range")
+	errTailTruncationOutOfRange = errors.New("history tail truncation out of range")
+)
+
+// truncateFromHead removes excess elements from the head of the freezer based
+// on the given parameters. It returns the number of items that were removed.
+func truncateFromHead(store ethdb.AncientStore, nhead uint64) (int, error) {
+	ohead, err := store.Ancients()
+	if err != nil {
+		return 0, err
+	}
+	otail, err := store.Tail()
+	if err != nil {
+		return 0, err
+	}
+	log.Info("Truncating from head", "ohead", ohead, "tail", otail, "nhead", nhead)
+
+	// Ensure that the truncation target falls within the valid range.
+	if ohead < nhead || nhead < otail {
+		return 0, fmt.Errorf("%w, tail: %d, head: %d, target: %d", errHeadTruncationOutOfRange, otail, ohead, nhead)
+	}
+	// Short circuit if nothing to truncate.
+	if ohead == nhead {
+		return 0, nil
+	}
+	ohead, err = store.TruncateHead(nhead)
+	if err != nil {
+		return 0, err
+	}
+	// Associated root->id mappings are left in the database and wait
+	// for overwriting.
+	return int(ohead - nhead), nil
+}
+
+// truncateFromTail removes excess elements from the end of the freezer based
+// on the given parameters. It returns the number of items that were removed.
+func truncateFromTail(store ethdb.AncientStore, ntail uint64) (int, error) {
+	ohead, err := store.Ancients()
+	if err != nil {
+		return 0, err
+	}
+	otail, err := store.Tail()
+	if err != nil {
+		return 0, err
+	}
+	// Ensure that the truncation target falls within the valid range.
+	if otail > ntail || ntail > ohead {
+		return 0, fmt.Errorf("%w, tail: %d, head: %d, target: %d", errTailTruncationOutOfRange, otail, ohead, ntail)
+	}
+	// Short circuit if nothing to truncate.
+	if otail == ntail {
+		return 0, nil
+	}
+	otail, err = store.TruncateTail(ntail)
+	if err != nil {
+		return 0, err
+	}
+	// Associated root->id mappings are left in the database.
+	return int(ntail - otail), nil
+}

triedb/pathdb/history_reader.go

@@ -320,11 +320,12 @@ func (r *historyReader) read(state stateIdentQuery, stateID uint64, lastID uint6
 	tail, err := r.freezer.Tail()
 	if err != nil {
 		return nil, err
-	}
-	// stateID == tail is allowed, as the first history object preserved
-	// is tail+1
+	} // firstID = tail+1
+
+	// stateID+1 == firstID is allowed, as all the subsequent state histories
+	// are present with no gap inside.
 	if stateID < tail {
-		return nil, errors.New("historical state has been pruned")
+		return nil, fmt.Errorf("historical state has been pruned, first: %d, state: %d", tail+1, stateID)
 	}
 
 	// To serve the request, all state histories from stateID+1 to lastID

triedb/pathdb/history_reader_test.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/internal/testrand"
 )
 
 func waitIndexing(db *Database) {
@@ -36,11 +37,29 @@ func waitIndexing(db *Database) {
 	}
 }
 
-func checkHistoricState(env *tester, root common.Hash, hr *historyReader) error {
+func stateAvail(id uint64, env *tester) bool {
+	if env.db.config.StateHistory == 0 {
+		return true
+	}
+	dl := env.db.tree.bottom()
+	if dl.stateID() <= env.db.config.StateHistory {
+		return true
+	}
+	firstID := dl.stateID() - env.db.config.StateHistory + 1
+
+	return id+1 >= firstID
+}
+
+func checkHistoricalState(env *tester, root common.Hash, id uint64, hr *historyReader) error {
+	if !stateAvail(id, env) {
+		return nil
+	}
+
 	// Short circuit if the historical state is no longer available
 	if rawdb.ReadStateID(env.db.diskdb, root) == nil {
-		return nil
+		return fmt.Errorf("state not found %d %x", id, root)
 	}
+
 	var (
 		dl       = env.db.tree.bottom()
 		stateID  = rawdb.ReadStateID(env.db.diskdb, root)
@@ -124,22 +143,22 @@ func testHistoryReader(t *testing.T, historyLimit uint64) {
 	defer func() {
 		maxDiffLayers = 128
 	}()
-	//log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelDebug, true)))
 
+	// log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelDebug, true)))
 	env := newTester(t, historyLimit, false, 64, true, "")
 	defer env.release()
 	waitIndexing(env.db)
 
 	var (
 		roots = env.roots
-		dRoot = env.db.tree.bottom().rootHash()
+		dl    = env.db.tree.bottom()
 		hr    = newHistoryReader(env.db.diskdb, env.db.stateFreezer)
 	)
-	for _, root := range roots {
-		if root == dRoot {
+	for i, root := range roots {
+		if root == dl.rootHash() {
 			break
 		}
-		if err := checkHistoricState(env, root, hr); err != nil {
+		if err := checkHistoricalState(env, root, uint64(i+1), hr); err != nil {
 			t.Fatal(err)
 		}
 	}
@@ -148,12 +167,41 @@ func testHistoryReader(t *testing.T, historyLimit uint64) {
 	env.extend(4)
 	waitIndexing(env.db)
 
-	for _, root := range roots {
-		if root == dRoot {
+	for i, root := range roots {
+		if root == dl.rootHash() {
 			break
 		}
-		if err := checkHistoricState(env, root, hr); err != nil {
+		if err := checkHistoricalState(env, root, uint64(i+1), hr); err != nil {
 			t.Fatal(err)
 		}
 	}
 }
+
+func TestHistoricalStateReader(t *testing.T) {
+	maxDiffLayers = 4
+	defer func() {
+		maxDiffLayers = 128
+	}()
+
+	//log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelDebug, true)))
+	env := newTester(t, 0, false, 64, true, "")
+	defer env.release()
+	waitIndexing(env.db)
+
+	// non-canonical state
+	fakeRoot := testrand.Hash()
+	rawdb.WriteStateID(env.db.diskdb, fakeRoot, 10)
+
+	_, err := env.db.HistoricReader(fakeRoot)
+	if err == nil {
+		t.Fatal("expected error")
+	}
+	t.Log(err)
+
+	// canonical state
+	realRoot := env.roots[9]
+	_, err = env.db.HistoricReader(realRoot)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+}

triedb/pathdb/history_state.go

@@ -504,6 +504,20 @@ func (h *stateHistory) decode(accountData, storageData, accountIndexes, storageI
 	return nil
 }
 
+// readStateHistoryMeta reads the metadata of state history with the specified id.
+func readStateHistoryMeta(reader ethdb.AncientReader, id uint64) (*meta, error) {
+	data := rawdb.ReadStateHistoryMeta(reader, id)
+	if len(data) == 0 {
+		return nil, fmt.Errorf("metadata is not found, %d", id)
+	}
+	var m meta
+	err := m.decode(data)
+	if err != nil {
+		return nil, err
+	}
+	return &m, nil
+}
+
 // readStateHistory reads a single state history records with the specified id.
 func readStateHistory(reader ethdb.AncientReader, id uint64) (*stateHistory, error) {
 	mData, accountIndexes, storageIndexes, accountData, storageData, err := rawdb.ReadStateHistory(reader, id)
@@ -568,8 +582,8 @@ func writeStateHistory(writer ethdb.AncientWriter, dl *diffLayer) error {
 	return nil
 }
 
-// checkStateHistories retrieves a batch of meta objects with the specified range
-// and performs the callback on each item.
+// checkStateHistories retrieves a batch of metadata objects with the specified
+// range and performs the callback on each item.
 func checkStateHistories(reader ethdb.AncientReader, start, count uint64, check func(*meta) error) error {
 	for count > 0 {
 		number := count
@@ -594,87 +608,3 @@ func checkStateHistories(reader ethdb.AncientReader, start, count uint64, check
 	}
 	return nil
 }
-
-// truncateFromHead removes the extra state histories from the head with the given
-// parameters. It returns the number of items removed from the head.
-func truncateFromHead(db ethdb.Batcher, store ethdb.AncientStore, nhead uint64) (int, error) {
-	ohead, err := store.Ancients()
-	if err != nil {
-		return 0, err
-	}
-	otail, err := store.Tail()
-	if err != nil {
-		return 0, err
-	}
-	// Ensure that the truncation target falls within the specified range.
-	if ohead < nhead || nhead < otail {
-		return 0, fmt.Errorf("out of range, tail: %d, head: %d, target: %d", otail, ohead, nhead)
-	}
-	// Short circuit if nothing to truncate.
-	if ohead == nhead {
-		return 0, nil
-	}
-	// Load the meta objects in range [nhead+1, ohead]
-	blobs, err := rawdb.ReadStateHistoryMetaList(store, nhead+1, ohead-nhead)
-	if err != nil {
-		return 0, err
-	}
-	batch := db.NewBatch()
-	for _, blob := range blobs {
-		var m meta
-		if err := m.decode(blob); err != nil {
-			return 0, err
-		}
-		rawdb.DeleteStateID(batch, m.root)
-	}
-	if err := batch.Write(); err != nil {
-		return 0, err
-	}
-	ohead, err = store.TruncateHead(nhead)
-	if err != nil {
-		return 0, err
-	}
-	return int(ohead - nhead), nil
-}
-
-// truncateFromTail removes the extra state histories from the tail with the given
-// parameters. It returns the number of items removed from the tail.
-func truncateFromTail(db ethdb.Batcher, store ethdb.AncientStore, ntail uint64) (int, error) {
-	ohead, err := store.Ancients()
-	if err != nil {
-		return 0, err
-	}
-	otail, err := store.Tail()
-	if err != nil {
-		return 0, err
-	}
-	// Ensure that the truncation target falls within the specified range.
-	if otail > ntail || ntail > ohead {
-		return 0, fmt.Errorf("out of range, tail: %d, head: %d, target: %d", otail, ohead, ntail)
-	}
-	// Short circuit if nothing to truncate.
-	if otail == ntail {
-		return 0, nil
-	}
-	// Load the meta objects in range [otail+1, ntail]
-	blobs, err := rawdb.ReadStateHistoryMetaList(store, otail+1, ntail-otail)
-	if err != nil {
-		return 0, err
-	}
-	batch := db.NewBatch()
-	for _, blob := range blobs {
-		var m meta
-		if err := m.decode(blob); err != nil {
-			return 0, err
-		}
-		rawdb.DeleteStateID(batch, m.root)
-	}
-	if err := batch.Write(); err != nil {
-		return 0, err
-	}
-	otail, err = store.TruncateTail(ntail)
-	if err != nil {
-		return 0, err
-	}
-	return int(ntail - otail), nil
-}