Make PruneExecutionDB not depend on consensusDB

Refactor PruneExecutionDB, more specifically findImportantRoots
so it does not depend on consensusDB, but only on executionDB

Signed-off-by: Igor Braga <5835477+bragaigor@users.noreply.github.com>

Author: bragaigor

changelog/bragaigor-nit-4266.md

@@ -0,0 +1,2 @@
+### Changed
+ - Make `PruneExecutionDB` only depend on `executionDB` by removing `consensusDB` dependency

cmd/pruning/pruning.go

@@ -21,17 +21,12 @@ import (
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/rpc"
 
-	"github.com/offchainlabs/nitro/arbnode/dataposter/storage"
-	"github.com/offchainlabs/nitro/arbnode/db/read"
-	"github.com/offchainlabs/nitro/arbnode/mel"
-	"github.com/offchainlabs/nitro/arbutil"
 	"github.com/offchainlabs/nitro/bold/protocol"
 	"github.com/offchainlabs/nitro/cmd/chaininfo"
 	"github.com/offchainlabs/nitro/cmd/conf"
 	"github.com/offchainlabs/nitro/execution/gethexec"
 	"github.com/offchainlabs/nitro/solgen/go/bridgegen"
 	"github.com/offchainlabs/nitro/solgen/go/rollupgen"
-	"github.com/offchainlabs/nitro/staker"
 	"github.com/offchainlabs/nitro/staker/bold"
 	legacystaker "github.com/offchainlabs/nitro/staker/legacy"
 	multiprotocolstaker "github.com/offchainlabs/nitro/staker/multi_protocol"
@@ -94,16 +89,6 @@ func findImportantRoots(ctx context.Context, executionDB ethdb.Database, stack *
 	if chainConfig == nil {
 		return nil, errors.New("database doesn't have a chain config (was this node initialized?)")
 	}
-	consensusDB, err := stack.OpenDatabaseWithOptions("arbitrumdata", node.DatabaseOptions{MetricsNamespace: "arbitrumdata/", ReadOnly: true, PebbleExtraOptions: persistentConfig.Pebble.ExtraOptions("arbitrumdata"), NoFreezer: true})
-	if err != nil {
-		return nil, err
-	}
-	defer func() {
-		err := consensusDB.Close()
-		if err != nil {
-			log.Warn("failed to close arbitrum database after finding pruning targets", "err", err)
-		}
-	}()
 	roots := importantRoots{
 		executionDB: executionDB,
 	}
@@ -113,7 +98,7 @@ func findImportantRoots(ctx context.Context, executionDB ethdb.Database, stack *
 	if genesisHeader == nil {
 		return nil, errors.New("missing L2 genesis block header")
 	}
-	err = roots.addHeader(genesisHeader, false)
+	err := roots.addHeader(genesisHeader, false)
 	if err != nil {
 		return nil, err
 	}
@@ -139,24 +124,25 @@ func findImportantRoots(ctx context.Context, executionDB ethdb.Database, stack *
 			log.Warn("missing latest confirmed block", "hash", confirmedHash)
 		}
 
-		validatorDB := rawdb.NewTable(consensusDB, storage.BlockValidatorPrefix)
-		lastValidated, err := staker.ReadLastValidatedInfo(validatorDB)
+		data, err := executionDB.Get(gethexec.ValidatedBlockHashKey)
 		if err != nil {
 			return nil, err
 		}
-		if lastValidated != nil {
+		lastValidatedBlockHash := common.BytesToHash(data)
+
+		if lastValidatedBlockHash != (common.Hash{}) {
 			var lastValidatedHeader *types.Header
-			headerNum, found := rawdb.ReadHeaderNumber(executionDB, lastValidated.GlobalState.BlockHash)
+			headerNum, found := rawdb.ReadHeaderNumber(executionDB, lastValidatedBlockHash)
 			if found {
-				lastValidatedHeader = rawdb.ReadHeader(executionDB, lastValidated.GlobalState.BlockHash, headerNum)
+				lastValidatedHeader = rawdb.ReadHeader(executionDB, lastValidatedBlockHash, headerNum)
 			}
 			if lastValidatedHeader != nil {
 				err = roots.addHeader(lastValidatedHeader, false)
 				if err != nil {
 					return nil, err
 				}
 			} else {
-				log.Warn("missing latest validated block", "hash", lastValidated.GlobalState.BlockHash)
+				log.Warn("missing latest validated block", "hash", lastValidatedBlockHash)
 			}
 		}
 	} else if initConfig.Prune == "full" || initConfig.Prune == "minimal" {
@@ -179,56 +165,21 @@ func findImportantRoots(ctx context.Context, executionDB ethdb.Database, stack *
 		return nil, fmt.Errorf("unknown pruning mode: \"%v\"", initConfig.Prune)
 	}
 	if initConfig.Prune != "minimal" && l1Client != nil {
-		// in pruning modes other then "minimal", find the latest finalized block and add it as a pruning target
-		l1Block, err := l1Client.BlockByNumber(ctx, big.NewInt(int64(rpc.FinalizedBlockNumber)))
-		if err != nil {
-			return nil, fmt.Errorf("failed to get finalized block: %w", err)
+		// in pruning modes other than "minimal", get the latest finalized block and add it as a pruning target
+		finalizedBlockHash := rawdb.ReadFinalizedBlockHash(executionDB)
+		finalizedBlockNumber, ok := rawdb.ReadHeaderNumber(executionDB, finalizedBlockHash)
+		if !ok {
+			return nil, errors.New("Number of finalized block is missing")
 		}
-		l1BlockNum := l1Block.NumberU64()
-		var batch uint64
-		if melEnabled {
-			batch, err = read.MELSequencerBatchCount(consensusDB)
+
+		l2Header := rawdb.ReadHeader(executionDB, finalizedBlockHash, finalizedBlockNumber)
+		if l2Header == nil {
+			log.Warn("latest finalized L2 block is unknown", "blockNum", finalizedBlockNumber)
 		} else {
-			batch, err = read.SequencerBatchCount(consensusDB)
-		}
-		if err != nil {
-			return nil, err
-		}
-		for {
-			if ctx.Err() != nil {
-				return nil, ctx.Err()
-			}
-			if batch == 0 {
-				// No batch has been finalized
-				break
-			}
-			batch -= 1
-			var meta mel.BatchMetadata
-			if melEnabled {
-				meta, err = read.MELBatchMetadata(consensusDB, batch)
-			} else {
-				meta, err = read.BatchMetadata(consensusDB, batch)
-			}
+			err = roots.addHeader(l2Header, false)
 			if err != nil {
 				return nil, err
 			}
-			if meta.ParentChainBlock <= l1BlockNum {
-				// #nosec G115
-				signedBlockNum := int64(arbutil.MessageIndexToBlockNumber(meta.MessageCount, genesisNum)) - 1
-				// #nosec G115
-				blockNum := uint64(signedBlockNum)
-				l2Hash := rawdb.ReadCanonicalHash(executionDB, blockNum)
-				l2Header := rawdb.ReadHeader(executionDB, l2Hash, blockNum)
-				if l2Header == nil {
-					log.Warn("latest finalized L2 block is unknown", "blockNum", signedBlockNum)
-					break
-				}
-				err = roots.addHeader(l2Header, false)
-				if err != nil {
-					return nil, err
-				}
-				break
-			}
 		}
 	}
 	roots.roots = append(roots.roots, common.Hash{}) // the latest snapshot

execution/gethexec/node.go

@@ -612,7 +612,7 @@ func (n *ExecutionNode) SetFinalityData(
 	finalizedFinalityData *arbutil.FinalityData,
 	validatedFinalityData *arbutil.FinalityData,
 ) containers.PromiseInterface[struct{}] {
-	err := n.SyncMonitor.SetFinalityData(safeFinalityData, finalizedFinalityData, validatedFinalityData)
+	err := n.SyncMonitor.SetFinalityData(n.ExecutionDB, safeFinalityData, finalizedFinalityData, validatedFinalityData)
 	if err != nil {
 		return containers.NewReadyPromise(struct{}{}, err)
 	}

execution/gethexec/sync_monitor.go

@@ -12,13 +12,16 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
 
 	"github.com/offchainlabs/nitro/arbutil"
 	"github.com/offchainlabs/nitro/consensus"
 	"github.com/offchainlabs/nitro/execution"
 )
 
+var ValidatedBlockHashKey = []byte("LastValidatedBlockHashKey")
+
 type syncDataEntry struct {
 	maxMessageCount arbutil.MessageIndex
 	timestamp       time.Time
@@ -274,6 +277,7 @@ func (s *SyncMonitor) getFinalityBlockHeader(
 }
 
 func (s *SyncMonitor) SetFinalityData(
+	executionDB ethdb.Database,
 	safeFinalityData *arbutil.FinalityData,
 	finalizedFinalityData *arbutil.FinalityData,
 	validatedFinalityData *arbutil.FinalityData,
@@ -288,6 +292,11 @@ func (s *SyncMonitor) SetFinalityData(
 	}
 	s.exec.bc.SetFinalized(finalizedBlockHeader)
 
+	if executionDB != nil && finalizedBlockHeader != nil {
+		finalizedBlockHash := finalizedBlockHeader.Hash()
+		executionDB.Put(ValidatedBlockHashKey, finalizedBlockHash.Bytes())
+	}
+
 	safeBlockHeader, err := s.getFinalityBlockHeader(
 		s.config.SafeBlockWaitForBlockValidator,
 		validatedFinalityData,

system_tests/finality_data_test.go

@@ -174,20 +174,20 @@ func TestFinalityDataWaitForBlockValidator(t *testing.T) {
 		BlockHash: validatedMsgResult.BlockHash,
 	}
 
-	err = builder.L2.ExecNode.SyncMonitor.SetFinalityData(&safeFinalityData, &finalizedFinalityData, &validatedFinalityData)
+	err = builder.L2.ExecNode.SyncMonitor.SetFinalityData(nil, &safeFinalityData, &finalizedFinalityData, &validatedFinalityData)
 	Require(t, err)
 
 	// wait for block validator is set to true in second node
 	checksFinalityData(t, "first node", ctx, builder.L2, validatedMsgIdx, validatedMsgIdx)
 
-	err = testClient2ndNode.ExecNode.SyncMonitor.SetFinalityData(&safeFinalityData, &finalizedFinalityData, &validatedFinalityData)
+	err = testClient2ndNode.ExecNode.SyncMonitor.SetFinalityData(nil, &safeFinalityData, &finalizedFinalityData, &validatedFinalityData)
 	Require(t, err)
 
 	// wait for block validator is no set to true in second node
 	checksFinalityData(t, "2nd node", ctx, testClient2ndNode, finalizedMsgIdx, safeMsgIdx)
 
 	// if validatedFinalityData is nil, error should be returned if waitForBlockValidator is set to true
-	err = builder.L2.ExecNode.SyncMonitor.SetFinalityData(&safeFinalityData, &finalizedFinalityData, nil)
+	err = builder.L2.ExecNode.SyncMonitor.SetFinalityData(nil, &safeFinalityData, &finalizedFinalityData, nil)
 	if err == nil {
 		t.Fatalf("err should not be nil")
 	}
@@ -247,7 +247,7 @@ func TestFinalityDataPushedFromConsensusToExecution(t *testing.T) {
 	ensureSafeBlockDoesNotExist(t, ctx, builder.L2, "first node after generating blocks")
 
 	// if nil is passed finality data should not be set
-	err := builder.L2.ExecNode.SyncMonitor.SetFinalityData(nil, nil, nil)
+	err := builder.L2.ExecNode.SyncMonitor.SetFinalityData(nil, nil, nil, nil)
 	Require(t, err)
 	ensureFinalizedBlockDoesNotExist(t, ctx, builder.L2, "first node after generating blocks and setting finality data to nil")
 	ensureSafeBlockDoesNotExist(t, ctx, builder.L2, "first node after generating blocks and setting finality data to nil")
@@ -310,7 +310,7 @@ func TestFinalityAfterReorg(t *testing.T) {
 		BlockHash: finalizedMsgResult.BlockHash,
 	}
 
-	err = builder.L2.ExecNode.SyncMonitor.SetFinalityData(&safeFinalityData, &finalizedFinalityData, nil)
+	err = builder.L2.ExecNode.SyncMonitor.SetFinalityData(nil, &safeFinalityData, &finalizedFinalityData, nil)
 	Require(t, err)
 
 	checksFinalityData(t, "before reorg", ctx, builder.L2, finalizedFinalityData.MsgIdx, safeFinalityData.MsgIdx)
@@ -360,7 +360,7 @@ func TestSetFinalityBlockHashMismatch(t *testing.T) {
 		BlockHash: common.Hash{},
 	}
 
-	err := builder.L2.ExecNode.SyncMonitor.SetFinalityData(&safeFinalityData, &finalizedFinalityData, nil)
+	err := builder.L2.ExecNode.SyncMonitor.SetFinalityData(nil, &safeFinalityData, &finalizedFinalityData, nil)
 	if err == nil {
 		t.Fatalf("err should not be nil")
 	}
@@ -408,13 +408,13 @@ func TestFinalityDataNodeOutOfSync(t *testing.T) {
 		BlockHash: finalizedMsgResult.BlockHash,
 	}
 
-	err = builder.L2.ExecNode.SyncMonitor.SetFinalityData(&safeFinalityData, &finalizedFinalityData, nil)
+	err = builder.L2.ExecNode.SyncMonitor.SetFinalityData(nil, &safeFinalityData, &finalizedFinalityData, nil)
 	Require(t, err)
 
 	checksFinalityData(t, "before out of sync", ctx, builder.L2, finalizedFinalityData.MsgIdx, safeFinalityData.MsgIdx)
 
 	// out of sync node
-	err = builder.L2.ExecNode.SyncMonitor.SetFinalityData(nil, nil, nil)
+	err = builder.L2.ExecNode.SyncMonitor.SetFinalityData(nil, nil, nil, nil)
 	Require(t, err)
 
 	ensureFinalizedBlockDoesNotExist(t, ctx, builder.L2, "out of sync")

system_tests/pruning_test.go

@@ -48,6 +48,9 @@ func testPruning(t *testing.T, mode string, pruneParallelStorageTraversal bool)
 	// PathScheme prunes the state trie by itself, so only HashScheme should be tested
 	builder.RequireScheme(t, rawdb.HashScheme)
 
+	// Needed to create safe blocks; hence forcing SetFinalityData call
+	builder.nodeConfig.ParentChainReader.UseFinalityData = true
+
 	_ = builder.Build(t)
 	l2cleanupDone := false
 	defer func() {
@@ -70,6 +73,14 @@ func testPruning(t *testing.T, mode string, pruneParallelStorageTraversal bool)
 	}
 	lastBlock, err := builder.L2.Client.BlockNumber(ctx)
 	Require(t, err)
+
+	// Cache both validated and finalized block hashes for l2 executionDB to later
+	// add to the new executionDB below
+	data, err := builder.L2.ExecNode.ExecutionDB.Get(gethexec.ValidatedBlockHashKey)
+	Require(t, err)
+	validatedBlockHash := common.BytesToHash(data)
+	finalizedBlockHash := rawdb.ReadFinalizedBlockHash(builder.L2.ExecNode.ExecutionDB)
+
 	l2cleanupDone = true
 	builder.L2.cleanup()
 	t.Log("stopped l2 node")
@@ -78,10 +89,16 @@ func testPruning(t *testing.T, mode string, pruneParallelStorageTraversal bool)
 		stack, err := node.New(builder.l2StackConfig)
 		Require(t, err)
 		defer stack.Close()
-		chainDB, err := stack.OpenDatabaseWithOptions("l2chaindata", node.DatabaseOptions{MetricsNamespace: "l2chaindata/", PebbleExtraOptions: conf.PersistentConfigDefault.Pebble.ExtraOptions("l2chaindata")})
+		executionDB, err := stack.OpenDatabaseWithOptions("l2chaindata", node.DatabaseOptions{MetricsNamespace: "l2chaindata/", PebbleExtraOptions: conf.PersistentConfigDefault.Pebble.ExtraOptions("l2chaindata")})
+		Require(t, err)
+		defer executionDB.Close()
+		executionDBEntriesBeforePruning := countStateEntries(executionDB)
+
+		// Since we're dealing with a new executionDB we store both validatedBlockHash and
+		// finalized blocks back into this new executionDB.
+		err = executionDB.Put(gethexec.ValidatedBlockHashKey, validatedBlockHash.Bytes())
 		Require(t, err)
-		defer chainDB.Close()
-		executionDBEntriesBeforePruning := countStateEntries(chainDB)
+		rawdb.WriteFinalizedBlockHash(executionDB, finalizedBlockHash)
 
 		prand := testhelpers.NewPseudoRandomDataSource(t, 1)
 		var testKeys [][]byte
@@ -90,11 +107,11 @@ func testPruning(t *testing.T, mode string, pruneParallelStorageTraversal bool)
 			testKeys = append(testKeys, prand.GetHash().Bytes())
 		}
 		for _, key := range testKeys {
-			err = chainDB.Put(key, common.FromHex("0xdeadbeef"))
+			err = executionDB.Put(key, common.FromHex("0xdeadbeef"))
 			Require(t, err)
 		}
 		for _, key := range testKeys {
-			if has, _ := chainDB.Has(key); !has {
+			if has, _ := executionDB.Has(key); !has {
 				Fatal(t, "internal test error - failed to check existence of test key")
 			}
 		}
@@ -104,16 +121,16 @@ func testPruning(t *testing.T, mode string, pruneParallelStorageTraversal bool)
 		initConfig.PruneParallelStorageTraversal = pruneParallelStorageTraversal
 		coreCacheConfig := gethexec.DefaultCacheConfigFor(&builder.execConfig.Caching)
 		persistentConfig := conf.PersistentConfigDefault
-		err = pruning.PruneExecutionDB(ctx, chainDB, stack, &initConfig, coreCacheConfig, &persistentConfig, builder.L1.Client, *builder.L2.ConsensusNode.DeployInfo, false, false)
+		err = pruning.PruneExecutionDB(ctx, executionDB, stack, &initConfig, coreCacheConfig, &persistentConfig, builder.L1.Client, *builder.L2.ConsensusNode.DeployInfo, false, false)
 		Require(t, err)
 
 		for _, key := range testKeys {
-			if has, _ := chainDB.Has(key); has {
+			if has, _ := executionDB.Has(key); has {
 				Fatal(t, "test key hasn't been pruned as expected")
 			}
 		}
 
-		executionDBEntriesAfterPruning := countStateEntries(chainDB)
+		executionDBEntriesAfterPruning := countStateEntries(executionDB)
 		t.Log("db entries pre-pruning:", executionDBEntriesBeforePruning)
 		t.Log("db entries post-pruning:", executionDBEntriesAfterPruning)