diff --git a/graft/coreth/core/blockchain_ext_test.go b/graft/coreth/core/blockchain_ext_test.go index fe83411f4c4b..731f5b4f9987 100644 --- a/graft/coreth/core/blockchain_ext_test.go +++ b/graft/coreth/core/blockchain_ext_test.go @@ -6,8 +6,6 @@ package core import ( "fmt" "math/big" - "os" - "path/filepath" "slices" "testing" @@ -25,6 +23,7 @@ import ( "github.com/ava-labs/avalanchego/graft/coreth/params" "github.com/ava-labs/avalanchego/graft/coreth/plugin/evm/customheader" "github.com/ava-labs/avalanchego/graft/coreth/plugin/evm/upgrade/ap4" + "github.com/ava-labs/avalanchego/graft/evm/utils/utilstest" ethparams "github.com/ava-labs/libevm/params" ) @@ -153,63 +152,6 @@ var reexecTests = []ReexecTest{ }, } -func copyMemDB(db ethdb.Database) (ethdb.Database, error) { - newDB := rawdb.NewMemoryDatabase() - iter := db.NewIterator(nil, nil) - defer iter.Release() - for iter.Next() { - if err := newDB.Put(iter.Key(), iter.Value()); err != nil { - return nil, err - } - } - - return newDB, nil -} - -// copyDir recursively copies all files and folders from a directory [src] to a -// new temporary directory and returns the path to the new directory. -func copyDir(t *testing.T, src string) string { - t.Helper() - - if src == "" { - return "" - } - - dst := t.TempDir() - err := filepath.Walk(src, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - // Calculate the relative path from src - relPath, err := filepath.Rel(src, path) - if err != nil { - return err - } - - // Skip the root directory itself - if relPath == "." { - return nil - } - - dstPath := filepath.Join(dst, relPath) - - if info.IsDir() { - return os.MkdirAll(dstPath, info.Mode().Perm()) - } - - data, err := os.ReadFile(path) - if err != nil { - return err - } - - return os.WriteFile(dstPath, data, info.Mode().Perm()) - }) - - require.NoError(t, err) - return dst -} - // checkBlockChainState creates a new BlockChain instance and checks that exporting each block from // genesis to last accepted from the original instance yields the same last accepted block and state // root. @@ -255,9 +197,8 @@ func checkBlockChainState( require.NoErrorf(checkState(acceptedState), "Check state failed for newly generated blockchain") // Copy the database over to prevent any issues when re-using [originalDB] after this call. - originalDB, err = copyMemDB(originalDB) - require.NoError(err) - newChainDataDir := copyDir(t, oldChainDataDir) + originalDB = utilstest.CopyEthDB(t, originalDB) + newChainDataDir := utilstest.CopyDir(t, oldChainDataDir) restartedChain, err := create(originalDB, gspec, lastAcceptedBlock.Hash(), newChainDataDir) require.NoError(err) defer restartedChain.Stop() @@ -1702,7 +1643,7 @@ func ReexecCorruptedStateTest(t *testing.T, create ReexecTestFunc) { blockchain.Stop() // Restart blockchain with existing state - newDir := copyDir(t, tempDir) // avoid file lock + newDir := utilstest.CopyDir(t, tempDir) // avoid file lock restartedBlockchain, err := create(chainDB, gspec, chain[1].Hash(), newDir, 4096) require.NoError(t, err) defer restartedBlockchain.Stop() diff --git a/graft/coreth/core/chain_makers.go b/graft/coreth/core/chain_makers.go index ef4abf9e9da6..4137f1a00cd5 100644 --- a/graft/coreth/core/chain_makers.go +++ b/graft/coreth/core/chain_makers.go @@ -32,7 +32,6 @@ import ( "math/big" "github.com/ava-labs/avalanchego/graft/coreth/consensus" - "github.com/ava-labs/avalanchego/graft/coreth/core/extstate" "github.com/ava-labs/avalanchego/graft/coreth/params" "github.com/ava-labs/avalanchego/graft/coreth/plugin/evm/customheader" "github.com/ava-labs/avalanchego/graft/coreth/plugin/evm/customtypes" @@ -43,6 +42,7 @@ import ( "github.com/ava-labs/libevm/core/types" "github.com/ava-labs/libevm/core/vm" "github.com/ava-labs/libevm/ethdb" + "github.com/ava-labs/libevm/libevm/stateconf" "github.com/ava-labs/libevm/triedb" "github.com/holiman/uint256" ) @@ -273,6 +273,12 @@ func (b *BlockGen) SetOnBlockGenerated(onBlockGenerated func(*types.Block)) { // values. Inserting them into BlockChain requires use of FakePow or // a similar non-validating proof of work implementation. func GenerateChain(config *params.ChainConfig, parent *types.Block, engine consensus.Engine, db ethdb.Database, n int, gap uint64, gen func(int, *BlockGen)) ([]*types.Block, []types.Receipts, error) { + stateCache := state.NewDatabase(db) + defer stateCache.TrieDB().Close() + return GenerateChainFromStateCache(config, parent, engine, stateCache, n, gap, gen) +} + +func GenerateChainFromStateCache(config *params.ChainConfig, parent *types.Block, engine consensus.Engine, stateCache state.Database, n int, gap uint64, gen func(int, *BlockGen)) ([]*types.Block, []types.Receipts, error) { if config == nil { config = params.TestChainConfig } @@ -301,7 +307,8 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse } // Write state changes to db - root, err := statedb.Commit(b.header.Number.Uint64(), config.IsEIP158(b.header.Number)) + statedbOpts := stateconf.WithTrieDBUpdateOpts(stateconf.WithTrieDBUpdatePayload(block.ParentHash(), block.Hash())) + root, err := statedb.Commit(b.header.Number.Uint64(), config.IsEIP158(b.header.Number), statedbOpts) if err != nil { panic(fmt.Sprintf("state write error: %v", err)) } @@ -314,16 +321,12 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse return block, b.receipts, nil } - // Forcibly use hash-based state scheme for retaining all nodes in disk. - triedb := triedb.NewDatabase(db, triedb.HashDefaults) - defer triedb.Close() - for i := 0; i < n; i++ { - statedb, err := state.New(parent.Root(), extstate.NewDatabaseWithNodeDB(db, triedb), nil) + statedb, err := state.New(parent.Root(), stateCache, nil) if err != nil { return nil, nil, err } - block, receipts, err := genblock(i, parent, triedb, statedb) + block, receipts, err := genblock(i, parent, stateCache.TrieDB(), statedb) if err != nil { return nil, nil, err } diff --git a/graft/coreth/network/network.go b/graft/coreth/network/network.go index 2e33bfe589c9..a596a9f0663c 100644 --- a/graft/coreth/network/network.go +++ b/graft/coreth/network/network.go @@ -41,6 +41,7 @@ var ( _ Network = (*network)(nil) _ validators.Connector = (*network)(nil) _ common.AppHandler = (*network)(nil) + _ p2p.NodeSampler = (*network)(nil) ) // SyncedNetworkClient defines ability to send request / response through the Network @@ -63,6 +64,7 @@ type SyncedNetworkClient interface { type Network interface { validators.Connector common.AppHandler + p2p.NodeSampler SyncedNetworkClient @@ -155,6 +157,24 @@ func NewNetwork( }, nil } +func (n *network) Sample(_ context.Context, limit int) []ids.NodeID { + if limit != 1 { + log.Warn("Sample called with limit > 1, but only 1 peer will be returned", "limit", limit) + } + + n.lock.Lock() + defer n.lock.Unlock() + node, ok, err := n.peers.GetAnyPeer(nil) + if err != nil { + log.Error("error getting peer from peer tracker", "error", err) + return nil + } + if !ok { + return nil + } + return []ids.NodeID{node} +} + // SendAppRequestAny synchronously sends request to an arbitrary peer with a // node version greater than or equal to minVersion. If minVersion is nil, // the request will be sent to any peer regardless of their version. diff --git a/graft/coreth/plugin/evm/atomic/sync/syncer_test.go b/graft/coreth/plugin/evm/atomic/sync/syncer_test.go index 2f111734c982..32212a7c2366 100644 --- a/graft/coreth/plugin/evm/atomic/sync/syncer_test.go +++ b/graft/coreth/plugin/evm/atomic/sync/syncer_test.go @@ -353,6 +353,7 @@ func setupTestInfrastructure(t *testing.T, serverTrieDB *triedb.Database) (conte handlers.NewLeafsRequestHandler(serverTrieDB, state.TrieKeyLength, nil, message.CorethCodec, handlerstats.NewNoopHandlerStats()), nil, nil, + nil, ) clientDB := versiondb.New(memdb.New()) diff --git a/graft/coreth/plugin/evm/config/config.md b/graft/coreth/plugin/evm/config/config.md index 4f8e66612ea8..8ed113fe567c 100644 --- a/graft/coreth/plugin/evm/config/config.md +++ b/graft/coreth/plugin/evm/config/config.md @@ -221,7 +221,7 @@ Configuration is provided as a JSON object. All fields are optional unless other | Option | Type | Description | Default | |--------|------|-------------|---------| -| `metrics-expensive-enabled` | bool | Enable expensive debug-level metrics; this includes Firewood metrics | `true` | +| `metrics-expensive-enabled` | bool | Enable expensive debug-level metrics | `true` | ## Security and Access @@ -259,7 +259,6 @@ Configuration is provided as a JSON object. All fields are optional unless other > **WARNING**: `firewood` and `path` schemes are untested in production. Using `path` is strongly discouraged. To use `firewood`, you must also set the following config options: > > - `populate-missing-tries: nil` -> - `state-sync-enabled: false` > - `snapshot-cache: 0` Failing to set these options will result in errors on VM initialization. Additionally, not all APIs are available - see these portions of the config documentation for more details. diff --git a/graft/coreth/plugin/evm/vm.go b/graft/coreth/plugin/evm/vm.go index 83e4699e2191..60ea9d817a6b 100644 --- a/graft/coreth/plugin/evm/vm.go +++ b/graft/coreth/plugin/evm/vm.go @@ -41,6 +41,7 @@ import ( "github.com/ava-labs/avalanchego/cache/metercacher" "github.com/ava-labs/avalanchego/codec" "github.com/ava-labs/avalanchego/database" + "github.com/ava-labs/avalanchego/database/merkle/firewood/syncer" "github.com/ava-labs/avalanchego/database/versiondb" "github.com/ava-labs/avalanchego/graft/coreth/consensus/dummy" "github.com/ava-labs/avalanchego/graft/coreth/core" @@ -58,6 +59,7 @@ import ( "github.com/ava-labs/avalanchego/graft/coreth/precompile/precompileconfig" "github.com/ava-labs/avalanchego/graft/coreth/warp" "github.com/ava-labs/avalanchego/graft/evm/constants" + "github.com/ava-labs/avalanchego/graft/evm/firewood" "github.com/ava-labs/avalanchego/graft/evm/message" "github.com/ava-labs/avalanchego/graft/evm/rpc" "github.com/ava-labs/avalanchego/graft/evm/sync/client" @@ -142,7 +144,6 @@ var ( errShuttingDownVM = errors.New("shutting down VM") errFirewoodSnapshotCacheDisabled = errors.New("snapshot cache must be disabled for Firewood") errFirewoodOfflinePruningUnsupported = errors.New("offline pruning is not supported for Firewood") - errFirewoodStateSyncUnsupported = errors.New("state sync is not yet supported for Firewood") errFirewoodMissingTrieRepopulationUnsupported = errors.New("missing trie repopulation is not supported for Firewood") ) @@ -400,9 +401,6 @@ func (vm *VM) Initialize( if vm.config.OfflinePruning { return errFirewoodOfflinePruningUnsupported } - if vm.config.StateSyncEnabled == nil || *vm.config.StateSyncEnabled { - return errFirewoodStateSyncUnsupported - } if vm.config.PopulateMissingTries != nil { return errFirewoodMissingTrieRepopulationUnsupported } @@ -582,13 +580,16 @@ func (vm *VM) initializeChain(lastAcceptedHash common.Hash) error { // If state sync is disabled, this function will wipe any ongoing summary from // disk to ensure that we do not continue syncing from an invalid snapshot. func (vm *VM) initializeStateSync(lastAcceptedHeight uint64) error { - // Create standalone EVM TrieDB (read only) for serving leafs requests. - // We create a standalone TrieDB here, so that it has a standalone cache from the one - // used by the node when processing blocks. - // However, Firewood does not support multiple TrieDBs, so we use the same one. - evmTrieDB := vm.eth.BlockChain().TrieDB() - if vm.ethConfig.StateScheme != customrawdb.FirewoodScheme { - evmTrieDB = triedb.NewDatabase( + leafHandlers := make(LeafHandlers) + leafMetricsNames := make(map[message.NodeType]string) + syncStats := handlerstats.GetOrRegisterHandlerStats(metrics.Enabled) + + switch scheme := vm.ethConfig.StateScheme; scheme { + case rawdb.HashScheme, "": + // Create standalone EVM TrieDB (read only) for serving leafs requests. + // We create a standalone TrieDB here, so that it has a standalone cache from the one + // used by the node when processing blocks. + evmTrieDB := triedb.NewDatabase( vm.chaindb, &triedb.Config{ DBOverride: hashdb.Config{ @@ -596,22 +597,34 @@ func (vm *VM) initializeStateSync(lastAcceptedHeight uint64) error { }.BackendConstructor, }, ) + // register default leaf request handler for state trie + stateLeafRequestConfig := &extension.LeafRequestConfig{ + LeafType: message.StateTrieNode, + MetricName: "sync_state_trie_leaves", + Handler: handlers.NewLeafsRequestHandler(evmTrieDB, + message.StateTrieKeyLength, + vm.blockChain, vm.networkCodec, + syncStats, + ), + } + leafHandlers[stateLeafRequestConfig.LeafType] = stateLeafRequestConfig.Handler + leafMetricsNames[stateLeafRequestConfig.LeafType] = stateLeafRequestConfig.MetricName + case customrawdb.FirewoodScheme: + tdb, ok := vm.eth.BlockChain().TrieDB().Backend().(*firewood.TrieDB) + if !ok { + return fmt.Errorf("expected a %T with %s scheme, got %T", tdb, customrawdb.FirewoodScheme, vm.eth.BlockChain().TrieDB().Backend()) + } + n := vm.Network.P2PNetwork() + if err := n.AddHandler(p2p.FirewoodRangeProofHandlerID, syncer.NewGetRangeProofHandler(tdb.Firewood)); err != nil { + return fmt.Errorf("adding firewood range proof handler: %w", err) + } + if err := n.AddHandler(p2p.FirewoodChangeProofHandlerID, syncer.NewGetChangeProofHandler(tdb.Firewood)); err != nil { + return fmt.Errorf("adding firewood change proof handler: %w", err) + } + default: + log.Warn("state sync is not supported for this scheme, no leaf handlers will be registered", "scheme", scheme) + return nil } - leafHandlers := make(LeafHandlers) - leafMetricsNames := make(map[message.NodeType]string) - // register default leaf request handler for state trie - syncStats := handlerstats.GetOrRegisterHandlerStats(metrics.Enabled) - stateLeafRequestConfig := &extension.LeafRequestConfig{ - LeafType: message.StateTrieNode, - MetricName: "sync_state_trie_leaves", - Handler: handlers.NewLeafsRequestHandler(evmTrieDB, - message.StateTrieKeyLength, - vm.blockChain, vm.networkCodec, - syncStats, - ), - } - leafHandlers[stateLeafRequestConfig.LeafType] = stateLeafRequestConfig.Handler - leafMetricsNames[stateLeafRequestConfig.LeafType] = stateLeafRequestConfig.MetricName extraLeafConfig := vm.extensionConfig.ExtraSyncLeafHandlerConfig if extraLeafConfig != nil { @@ -649,6 +662,7 @@ func (vm *VM) initializeStateSync(lastAcceptedHeight uint64) error { StateSyncDone: vm.stateSyncDone, Chain: newChainContextAdapter(vm.eth), State: vm.State, + SnowCtx: vm.ctx, Client: client.New( &client.Config{ Network: vm.Network, diff --git a/graft/coreth/plugin/evm/vmtest/test_syncervm.go b/graft/coreth/plugin/evm/vmtest/test_syncervm.go index d3fdbdcbe135..e9eb1e7dfad2 100644 --- a/graft/coreth/plugin/evm/vmtest/test_syncervm.go +++ b/graft/coreth/plugin/evm/vmtest/test_syncervm.go @@ -4,6 +4,7 @@ package vmtest import ( + "bytes" "context" "fmt" "math/big" @@ -19,16 +20,19 @@ import ( "github.com/ava-labs/libevm/log" "github.com/ava-labs/libevm/rlp" "github.com/ava-labs/libevm/trie" + "github.com/ava-labs/libevm/triedb" "github.com/stretchr/testify/require" "github.com/ava-labs/avalanchego/database/prefixdb" "github.com/ava-labs/avalanchego/graft/coreth/consensus/dummy" "github.com/ava-labs/avalanchego/graft/coreth/core" "github.com/ava-labs/avalanchego/graft/coreth/core/coretest" + "github.com/ava-labs/avalanchego/graft/coreth/core/extstate" "github.com/ava-labs/avalanchego/graft/coreth/params/paramstest" "github.com/ava-labs/avalanchego/graft/coreth/plugin/evm/customtypes" "github.com/ava-labs/avalanchego/graft/coreth/plugin/evm/extension" "github.com/ava-labs/avalanchego/graft/evm/constants" + "github.com/ava-labs/avalanchego/graft/evm/firewood" "github.com/ava-labs/avalanchego/graft/evm/sync/client" "github.com/ava-labs/avalanchego/graft/evm/sync/engine" "github.com/ava-labs/avalanchego/graft/evm/sync/synctest" @@ -50,6 +54,8 @@ import ( commonEng "github.com/ava-labs/avalanchego/snow/engine/common" ) +var schemes = []string{rawdb.HashScheme, customrawdb.FirewoodScheme} + type SyncerVMTest struct { Name string TestFunc func( @@ -87,9 +93,14 @@ func SkipStateSyncTest(t *testing.T, testSetup *SyncTestSetup) { StateSyncMinBlocks: 300, // must be greater than [syncableInterval] to skip sync SyncMode: block.StateSyncSkipped, } - testSyncVMSetup := initSyncServerAndClientVMs(t, test, engine.BlocksToFetch, testSetup) - testSyncerVM(t, testSyncVMSetup, test, testSetup.ExtraSyncerVMTest) + for _, scheme := range schemes { + test.StateScheme = scheme + t.Run(scheme, func(t *testing.T) { + testSyncVMSetup := initSyncServerAndClientVMs(t, test, engine.BlocksToFetch, testSetup) + testSyncerVM(t, testSyncVMSetup, test, testSetup.ExtraSyncerVMTest) + }) + } } func StateSyncFromScratchTest(t *testing.T, testSetup *SyncTestSetup) { @@ -98,9 +109,14 @@ func StateSyncFromScratchTest(t *testing.T, testSetup *SyncTestSetup) { StateSyncMinBlocks: 50, // must be less than [syncableInterval] to perform sync SyncMode: block.StateSyncStatic, } - testSyncVMSetup := initSyncServerAndClientVMs(t, test, engine.BlocksToFetch, testSetup) - testSyncerVM(t, testSyncVMSetup, test, testSetup.ExtraSyncerVMTest) + for _, scheme := range schemes { + test.StateScheme = scheme + t.Run(scheme, func(t *testing.T) { + testSyncVMSetup := initSyncServerAndClientVMs(t, test, engine.BlocksToFetch, testSetup) + testSyncerVM(t, testSyncVMSetup, test, testSetup.ExtraSyncerVMTest) + }) + } } func StateSyncFromScratchExceedParentTest(t *testing.T, testSetup *SyncTestSetup) { @@ -110,9 +126,14 @@ func StateSyncFromScratchExceedParentTest(t *testing.T, testSetup *SyncTestSetup StateSyncMinBlocks: 50, // must be less than [syncableInterval] to perform sync SyncMode: block.StateSyncStatic, } - testSyncVMSetup := initSyncServerAndClientVMs(t, test, int(numToGen), testSetup) - testSyncerVM(t, testSyncVMSetup, test, testSetup.ExtraSyncerVMTest) + for _, scheme := range schemes { + test.StateScheme = scheme + t.Run(scheme, func(t *testing.T) { + testSyncVMSetup := initSyncServerAndClientVMs(t, test, int(numToGen), testSetup) + testSyncerVM(t, testSyncVMSetup, test, testSetup.ExtraSyncerVMTest) + }) + } } func StateSyncToggleEnabledToDisabledTest(t *testing.T, testSetup *SyncTestSetup) { @@ -298,12 +319,13 @@ func initSyncServerAndClientVMs(t *testing.T, test SyncTestParams, numBlocks int // override commitInterval so the call to trie creates a commit at the height [syncableInterval]. // This is necessary to support fetching a state summary. - config := fmt.Sprintf(`{"commit-interval": %d, "state-sync-commit-interval": %d}`, test.SyncableInterval, test.SyncableInterval) + config := fmt.Sprintf(`{"commit-interval": %d, "state-history": %d, "state-sync-commit-interval": %d}`, test.SyncableInterval, test.SyncableInterval, test.SyncableInterval) serverVM, cb := testSetup.NewVM() fork := upgradetest.Latest serverTest := SetupTestVM(t, serverVM, TestVMConfig{ Fork: &fork, ConfigJSON: config, + Scheme: test.StateScheme, }) t.Cleanup(func() { log.Info("Shutting down server VM") @@ -324,8 +346,24 @@ func initSyncServerAndClientVMs(t *testing.T, test SyncTestParams, numBlocks int generateAndAcceptBlocks(t, serverVM, numBlocks, testSetup.GenFn, nil, cb) // make some accounts - r := rand.New(rand.NewSource(1)) - root, accounts := synctest.FillAccountsWithOverlappingStorage(t, r, serverVM.Ethereum().BlockChain().StateCache(), types.EmptyRootHash, 1000, 16) + var ( + r = rand.New(rand.NewSource(1)) + currentRoot = serverVM.Ethereum().BlockChain().LastAcceptedBlock().Root() + root common.Hash + fundedAccounts, allAccounts map[*utilstest.Key]*types.StateAccount + ) + if test.StateScheme == customrawdb.FirewoodScheme { + tdb, ok := serverVM.Ethereum().BlockChain().TrieDB().Backend().(*firewood.TrieDB) + require.True(ok) + tdb.SetHashAndHeight(common.Hash{}, 0) // must be set for FillAccountsWithStorageAndCode to work + } + root, allAccounts = synctest.FillAccountsWithStorageAndCode(t, r, serverVM.Ethereum().BlockChain().StateCache(), currentRoot, 1000) + fundedAccounts = make(map[*utilstest.Key]*types.StateAccount) + for key, account := range allAccounts { + if len(account.CodeHash) == 0 || bytes.Equal(account.CodeHash, types.EmptyCodeHash[:]) { + fundedAccounts[key] = account + } + } // patch serverVM's lastAcceptedBlock to have the new root // and update the vm's state so the trie with accounts will @@ -350,6 +388,7 @@ func initSyncServerAndClientVMs(t *testing.T, test SyncTestParams, numBlocks int syncerTest := SetupTestVM(t, syncerVM, TestVMConfig{ Fork: &fork, ConfigJSON: stateSyncEnabledJSON, + Scheme: test.StateScheme, IsSyncing: true, }) shutdownOnceSyncerVM := &shutdownOnceVM{InnerVM: syncerVM} @@ -415,7 +454,7 @@ func initSyncServerAndClientVMs(t *testing.T, test SyncTestParams, numBlocks int AppSender: serverTest.AppSender, SnowCtx: serverTest.Ctx, }, - fundedAccounts: accounts, + fundedAccounts: fundedAccounts, syncerVM: syncerVMSetup, } } @@ -460,6 +499,7 @@ type SyncTestParams struct { StateSyncMinBlocks uint64 SyncableInterval uint64 SyncMode block.StateSyncMode + StateScheme string expectedErr error } @@ -628,6 +668,8 @@ func generateAndAcceptBlocks(t *testing.T, vm extension.InnerVM, numBlocks int, // acceptExternalBlock defines a function to parse, verify, and accept a block once it has been // generated by GenerateChain acceptExternalBlock := func(block *types.Block) { + p := vm.Ethereum().BlockChain().GetBlock(block.ParentHash(), block.NumberU64()-1) + require.True(vm.Ethereum().BlockChain().HasState(p.Root()), "unavailable state for block %d with root %s", block.NumberU64(), block.Root().Hex()) bytes, err := rlp.EncodeToBytes(block) require.NoError(err) extendedBlock, err := vm.ParseBlock(t.Context(), bytes) @@ -639,11 +681,25 @@ func generateAndAcceptBlocks(t *testing.T, vm extension.InnerVM, numBlocks int, accepted(block) } } - _, _, err := core.GenerateChain( + + lastAccepted := vm.Ethereum().BlockChain().LastAcceptedBlock() + c := vm.Ethereum().BlockChain().StateCache() + + // Firewood's state lives on disk, so we must copy the entire state into a new state cache, unlike HashDB. + if vm.Config().StateScheme == customrawdb.FirewoodScheme { + memdb := utilstest.CopyEthDB(t, vm.Ethereum().ChainDb()) + tdb := triedb.NewDatabase(memdb, &triedb.Config{ + DBOverride: firewood.DefaultConfig(utilstest.CopyDir(t, vm.Ethereum().BlockChain().CacheConfig().ChainDataDir)).BackendConstructor, + }) + tdb.Backend().(*firewood.TrieDB).SetHashAndHeight(lastAccepted.Hash(), lastAccepted.NumberU64()) + c = extstate.NewDatabaseWithNodeDB(memdb, tdb) + } + + _, _, err := core.GenerateChainFromStateCache( vm.Ethereum().BlockChain().Config(), - vm.Ethereum().BlockChain().LastAcceptedBlock(), + lastAccepted, dummy.NewFakerWithCallbacks(cb), - vm.Ethereum().ChainDb(), + c, numBlocks, 10, func(i int, g *core.BlockGen) { diff --git a/graft/coreth/plugin/evm/vmtest/test_vm.go b/graft/coreth/plugin/evm/vmtest/test_vm.go index c131b6b7c3ee..5c998b97e1c2 100644 --- a/graft/coreth/plugin/evm/vmtest/test_vm.go +++ b/graft/coreth/plugin/evm/vmtest/test_vm.go @@ -125,7 +125,6 @@ func OverrideSchemeConfig(scheme string, configJSON string) (string, error) { // Set Firewood-specific configuration flags (these will override any existing values) configMap["state-scheme"] = customrawdb.FirewoodScheme configMap["snapshot-cache"] = 0 - configMap["state-sync-enabled"] = false // Marshal back to JSON result, err := json.Marshal(configMap) diff --git a/graft/evm/firewood/triedb.go b/graft/evm/firewood/triedb.go index 80f4bdb58085..ae62f61a5d1f 100644 --- a/graft/evm/firewood/triedb.go +++ b/graft/evm/firewood/triedb.go @@ -211,6 +211,11 @@ func (t *TrieDB) SetHashAndHeight(blockHash common.Hash, height uint64) { clear(t.tree.blockHashes) t.tree.blockHashes[blockHash] = struct{}{} t.tree.height = height + root, err := t.Firewood.Root() + if err != nil { + log.Error("get root in SetHashAndHeight", "error", err) + } + t.tree.root = common.Hash(root) } // Scheme returns the scheme of the database. @@ -446,9 +451,9 @@ func (t *TrieDB) createProposal(parent *proposal, ops []ffi.BatchOp) (*proposal, return nil, fmt.Errorf("create proposal from parent root %s: %w", parent.root.Hex(), err) } - // Edge case: genesis block + // Edge case: we know the genesis block has an empty parent hash. block := parent.height + 1 - if _, ok := parent.blockHashes[common.Hash{}]; ok && parent.root == types.EmptyRootHash { + if _, ok := parent.blockHashes[common.Hash{}]; ok && parent.height == 0 { block = 0 } diff --git a/graft/evm/sync/block/syncer_test.go b/graft/evm/sync/block/syncer_test.go index b4af06853c37..757e42671c12 100644 --- a/graft/evm/sync/block/syncer_test.go +++ b/graft/evm/sync/block/syncer_test.go @@ -174,6 +174,7 @@ func newTestEnvironment(t *testing.T, numBlocks int, c codec.Manager) *testEnvir nil, nil, blockHandler, + nil, ), } } diff --git a/graft/evm/sync/client/client.go b/graft/evm/sync/client/client.go index 19d612bffaaa..8943b46d0e48 100644 --- a/graft/evm/sync/client/client.go +++ b/graft/evm/sync/client/client.go @@ -23,6 +23,7 @@ import ( "github.com/ava-labs/avalanchego/graft/evm/message" "github.com/ava-labs/avalanchego/graft/evm/sync/client/stats" "github.com/ava-labs/avalanchego/ids" + "github.com/ava-labs/avalanchego/network/p2p" "github.com/ava-labs/avalanchego/version" ethparams "github.com/ava-labs/libevm/params" @@ -54,6 +55,8 @@ var _ Client = (*client)(nil) // Network defines the interface for sending sync requests over the network. // This interface is implemented by the network layer in coreth and subnet-evm. type Network interface { + p2p.NodeSampler + // SendSyncedAppRequestAny synchronously sends request to an arbitrary peer with a // node version greater than or equal to minVersion. // Returns response bytes, the ID of the chosen peer, and ErrRequestFailed if @@ -67,6 +70,9 @@ type Network interface { // TrackBandwidth should be called after receiving a response from a peer to // track performance. This is used to prioritize peers that are more responsive. TrackBandwidth(nodeID ids.NodeID, bandwidth float64) + + // P2PNetwork returns the unabstracted [p2p.Network]. + P2PNetwork() *p2p.Network } // Client synchronously fetches data from the network to fulfill state sync requests. @@ -82,6 +88,12 @@ type Client interface { // GetCode synchronously retrieves code associated with the given hashes GetCode(ctx context.Context, hashes []common.Hash) ([][]byte, error) + + // AddClient creates a separate client on the underlying [p2p.Network]. + AddClient(handlerID uint64) *p2p.Client + + // StateSyncNodes returns the list of nodes provided via config. + StateSyncNodes() []ids.NodeID } // parseResponseFn parses given response bytes in context of specified request @@ -121,6 +133,14 @@ func New(config *Config) *client { } } +func (c *client) AddClient(handlerID uint64) *p2p.Client { + return c.network.P2PNetwork().NewClient(handlerID, c.network) +} + +func (c *client) StateSyncNodes() []ids.NodeID { + return c.stateSyncNodes +} + // GetLeafs synchronously retrieves leafs as per given [message.LeafsRequest] // Retries when: // - response bytes could not be unmarshalled to [message.LeafsResponse] diff --git a/graft/evm/sync/client/test_client.go b/graft/evm/sync/client/test_client.go index 42a658cb71f1..c54628fdb60e 100644 --- a/graft/evm/sync/client/test_client.go +++ b/graft/evm/sync/client/test_client.go @@ -16,6 +16,7 @@ import ( "github.com/ava-labs/avalanchego/graft/evm/message" "github.com/ava-labs/avalanchego/graft/evm/sync/handlers" "github.com/ava-labs/avalanchego/ids" + "github.com/ava-labs/avalanchego/network/p2p" ) var ( @@ -40,6 +41,8 @@ type TestClient struct { // GetBlocksIntercept is called on every GetBlocks request if set to a non-nil callback. // The returned response will be returned by TestClient to the caller. GetBlocksIntercept func(blockReq message.BlockRequest, blocks types.Blocks) (types.Blocks, error) + + clients map[uint64]*p2p.Client } func NewTestClient( @@ -47,13 +50,27 @@ func NewTestClient( leafHandler handlers.LeafRequestHandler, codesHandler *handlers.CodeRequestHandler, blocksHandler *handlers.BlockRequestHandler, + clients map[uint64]*p2p.Client, ) *TestClient { return &TestClient{ codec: codec, leafsHandler: leafHandler, codesHandler: codesHandler, blocksHandler: blocksHandler, + clients: clients, + } +} + +func (ml *TestClient) AddClient(handlerID uint64) *p2p.Client { + client, exists := ml.clients[handlerID] + if !exists { + panic(fmt.Sprintf("no client for handler ID %d", handlerID)) } + return client +} + +func (*TestClient) StateSyncNodes() []ids.NodeID { + return nil } func (ml *TestClient) GetLeafs(ctx context.Context, request message.LeafsRequest) (message.LeafsResponse, error) { diff --git a/graft/evm/sync/client/test_network.go b/graft/evm/sync/client/test_network.go index ef4da67c0c31..4e0cff219df8 100644 --- a/graft/evm/sync/client/test_network.go +++ b/graft/evm/sync/client/test_network.go @@ -8,6 +8,7 @@ import ( "errors" "github.com/ava-labs/avalanchego/ids" + "github.com/ava-labs/avalanchego/network/p2p" "github.com/ava-labs/avalanchego/version" ) @@ -24,6 +25,14 @@ type testNetwork struct { nodesRequested []ids.NodeID } +func (*testNetwork) P2PNetwork() *p2p.Network { + panic("P2PNetwork unimplemented") +} + +func (*testNetwork) Sample(context.Context, int) []ids.NodeID { + panic("Sample unimplemented") +} + func (t *testNetwork) SendSyncedAppRequestAny(_ context.Context, _ *version.Application, _ []byte) ([]byte, ids.NodeID, error) { if len(t.response) == 0 { return nil, ids.EmptyNodeID, errors.New("no tested response to return in testNetwork") diff --git a/graft/evm/sync/code/syncer_test.go b/graft/evm/sync/code/syncer_test.go index cf820f9fee07..97917825a519 100644 --- a/graft/evm/sync/code/syncer_test.go +++ b/graft/evm/sync/code/syncer_test.go @@ -47,7 +47,7 @@ func testCodeSyncer(t *testing.T, test codeSyncerTest, c codec.Manager) { // Set up mockClient codeRequestHandler := handlers.NewCodeRequestHandler(serverDB, c, handlerstats.NewNoopHandlerStats()) - mockClient := client.NewTestClient(c, nil, codeRequestHandler, nil) + mockClient := client.NewTestClient(c, nil, codeRequestHandler, nil, nil) mockClient.GetCodeIntercept = test.getCodeIntercept clientDB := test.clientDB diff --git a/graft/evm/sync/engine/client.go b/graft/evm/sync/engine/client.go index cb3a81098b12..c3d7ee661f33 100644 --- a/graft/evm/sync/engine/client.go +++ b/graft/evm/sync/engine/client.go @@ -14,14 +14,18 @@ import ( "github.com/ava-labs/libevm/log" "github.com/ava-labs/libevm/params" + "github.com/ava-labs/avalanchego/api/metrics" "github.com/ava-labs/avalanchego/database" + "github.com/ava-labs/avalanchego/database/merkle/firewood/syncer" "github.com/ava-labs/avalanchego/database/versiondb" "github.com/ava-labs/avalanchego/graft/evm/core/state/snapshot" + "github.com/ava-labs/avalanchego/graft/evm/firewood" "github.com/ava-labs/avalanchego/graft/evm/message" "github.com/ava-labs/avalanchego/graft/evm/sync/code" "github.com/ava-labs/avalanchego/graft/evm/sync/evmstate" "github.com/ava-labs/avalanchego/graft/evm/sync/types" "github.com/ava-labs/avalanchego/ids" + "github.com/ava-labs/avalanchego/snow" "github.com/ava-labs/avalanchego/snow/engine/snowman/block" "github.com/ava-labs/avalanchego/vms/components/chain" @@ -97,6 +101,7 @@ type ClientConfig struct { Acceptor BlockAcceptor VerDB *versiondb.Database MetadataDB database.Database + SnowCtx *snow.Context // Extension points. SyncSummaryProvider message.SyncSummaryProvider @@ -394,14 +399,36 @@ func (c *client) newSyncerRegistry(summary message.Syncable) (*SyncerRegistry, e return nil, fmt.Errorf("failed to create code syncer: %w", err) } - stateSyncer, err := evmstate.NewSyncer( - c.config.Client, c.config.ChainDB, - summary.GetBlockRoot(), - codeQueue, c.config.RequestSize, - evmstate.WithLeafsRequestType(c.config.LeafsRequestType), - ) - if err != nil { - return nil, fmt.Errorf("failed to create EVM state syncer: %w", err) + var stateSyncer types.Syncer + if tdb, ok := c.config.Chain.BlockChain().TrieDB().Backend().(*firewood.TrieDB); ok { + registerer, err := metrics.MakeAndRegister(c.config.SnowCtx.Metrics, "sync_firewood") + if err != nil { + return nil, fmt.Errorf("failed to create firewood syncer metrics registerer: %w", err) + } + stateSyncer, err = evmstate.NewFirewoodSyncer( + syncer.Config{ + Log: c.config.SnowCtx.Log, + Registerer: registerer, + StateSyncNodes: c.config.Client.StateSyncNodes(), + }, + tdb.Firewood, + summary.GetBlockRoot(), + codeQueue, + c.config.Client, + ) + if err != nil { + return nil, fmt.Errorf("failed to create firewood syncer: %w", err) + } + } else { + stateSyncer, err = evmstate.NewSyncer( + c.config.Client, c.config.ChainDB, + summary.GetBlockRoot(), + codeQueue, c.config.RequestSize, + evmstate.WithLeafsRequestType(c.config.LeafsRequestType), + ) + if err != nil { + return nil, fmt.Errorf("failed to create EVM state syncer: %w", err) + } } syncers := []types.Syncer{blockSyncer, codeSyncer, stateSyncer} diff --git a/graft/evm/sync/engine/server.go b/graft/evm/sync/engine/server.go index a147ce05ec5c..8c13af41e8c2 100644 --- a/graft/evm/sync/engine/server.go +++ b/graft/evm/sync/engine/server.go @@ -11,6 +11,7 @@ import ( "github.com/ava-labs/libevm/common" "github.com/ava-labs/libevm/core/types" "github.com/ava-labs/libevm/log" + "github.com/ava-labs/libevm/triedb" "github.com/ava-labs/avalanchego/database" "github.com/ava-labs/avalanchego/snow/engine/snowman/block" @@ -32,6 +33,9 @@ type BlockChain interface { // ResetToStateSyncedBlock resets the blockchain to the given synced block. ResetToStateSyncedBlock(block *types.Block) error + + // TrieDB returns the database used for storing the state trie. + TrieDB() *triedb.Database } // SummaryProvider provides state summaries for blocks. diff --git a/graft/evm/sync/evmstate/firewood_syncer.go b/graft/evm/sync/evmstate/firewood_syncer.go index 99d83b8f2165..d0780c46ecb2 100644 --- a/graft/evm/sync/evmstate/firewood_syncer.go +++ b/graft/evm/sync/evmstate/firewood_syncer.go @@ -12,6 +12,7 @@ import ( "github.com/ava-labs/libevm/common" "github.com/ava-labs/avalanchego/database/merkle/firewood/syncer" + "github.com/ava-labs/avalanchego/graft/evm/sync/client" "github.com/ava-labs/avalanchego/graft/evm/sync/code" "github.com/ava-labs/avalanchego/graft/evm/sync/types" "github.com/ava-labs/avalanchego/ids" @@ -32,14 +33,14 @@ type FirewoodSyncer struct { finalizeOnce func() error } -func NewFirewoodSyncer(config syncer.Config, db *ffi.Database, target common.Hash, codeQueue *code.Queue, rangeProofClient, changeProofClient *p2p.Client) (*FirewoodSyncer, error) { +func NewFirewoodSyncer(config syncer.Config, db *ffi.Database, target common.Hash, codeQueue *code.Queue, client client.Client) (*FirewoodSyncer, error) { s, err := syncer.NewEVM( config, db, codeQueue, ids.ID(target), - rangeProofClient, - changeProofClient, + client.AddClient(p2p.FirewoodRangeProofHandlerID), + client.AddClient(p2p.FirewoodChangeProofHandlerID), ) if err != nil { return nil, err diff --git a/graft/evm/sync/evmstate/firewood_syncer_test.go b/graft/evm/sync/evmstate/firewood_syncer_test.go index 060390016b3e..58207e77e884 100644 --- a/graft/evm/sync/evmstate/firewood_syncer_test.go +++ b/graft/evm/sync/evmstate/firewood_syncer_test.go @@ -27,6 +27,7 @@ import ( "github.com/ava-labs/avalanchego/graft/evm/sync/synctest" "github.com/ava-labs/avalanchego/graft/evm/utils/utilstest" "github.com/ava-labs/avalanchego/ids" + "github.com/ava-labs/avalanchego/network/p2p" "github.com/ava-labs/avalanchego/network/p2p/p2ptest" "github.com/ava-labs/avalanchego/vms/evm/sync/customrawdb" @@ -147,10 +148,12 @@ func createSyncers(t *testing.T, clientState, serverState state.Database, root c // so the codec choice only affects the code request handler which is auxiliary to these tests. var ( codeRequestHandler = handlers.NewCodeRequestHandler(serverState.DiskDB(), message.CorethCodec, handlerstats.NewNoopHandlerStats()) - mockClient = statesyncclient.NewTestClient(message.CorethCodec, nil, codeRequestHandler, nil) serverDB = dbFromState(t, serverState) - rHandler = p2ptest.NewSelfClient(t, t.Context(), ids.EmptyNodeID, syncer.NewGetRangeProofHandler(serverDB)) - cHandler = p2ptest.NewSelfClient(t, t.Context(), ids.EmptyNodeID, syncer.NewGetChangeProofHandler(serverDB)) + clients = map[uint64]*p2p.Client{ + p2p.FirewoodRangeProofHandlerID: p2ptest.NewSelfClient(t, t.Context(), ids.EmptyNodeID, syncer.NewGetRangeProofHandler(serverDB)), + p2p.FirewoodChangeProofHandlerID: p2ptest.NewSelfClient(t, t.Context(), ids.EmptyNodeID, syncer.NewGetChangeProofHandler(serverDB)), + } + mockClient = statesyncclient.NewTestClient(message.CorethCodec, nil, codeRequestHandler, nil, clients) ) // Create the producer code queue. @@ -167,8 +170,7 @@ func createSyncers(t *testing.T, clientState, serverState state.Database, root c dbFromState(t, clientState), root, codeQueue, - rHandler, - cHandler, + mockClient, ) require.NoError(t, err, "NewFirewoodSyncer()") return firewoodSyncer, codeSyncer, codeQueue diff --git a/graft/evm/sync/evmstate/sync_test.go b/graft/evm/sync/evmstate/sync_test.go index f15a42161cc0..a02483af623d 100644 --- a/graft/evm/sync/evmstate/sync_test.go +++ b/graft/evm/sync/evmstate/sync_test.go @@ -60,7 +60,7 @@ func testSync(t *testing.T, test syncTest, c codec.Manager) { leafsRequestHandler := handlers.NewLeafsRequestHandler(serverDB.TrieDB(), message.StateTrieKeyLength, nil, c, handlerstats.NewNoopHandlerStats()) codeRequestHandler := handlers.NewCodeRequestHandler(serverDB.DiskDB(), c, handlerstats.NewNoopHandlerStats()) - mockClient := client.NewTestClient(c, leafsRequestHandler, codeRequestHandler, nil) + mockClient := client.NewTestClient(c, leafsRequestHandler, codeRequestHandler, nil, nil) // Set intercept functions for the mock client mockClient.GetLeafsIntercept = test.GetLeafsIntercept mockClient.GetCodeIntercept = test.GetCodeIntercept diff --git a/graft/evm/sync/synctest/trie.go b/graft/evm/sync/synctest/trie.go index 3a7a305be49f..a75d10c429b2 100644 --- a/graft/evm/sync/synctest/trie.go +++ b/graft/evm/sync/synctest/trie.go @@ -210,13 +210,15 @@ func FillAccounts( func FillAccountsWithStorageAndCode(t *testing.T, r *rand.Rand, serverDB state.Database, root common.Hash, numAccounts int) (common.Hash, map[*utilstest.Key]*types.StateAccount) { return FillAccounts(t, r, serverDB, root, numAccounts, func(t *testing.T, _ int, addr common.Address, account types.StateAccount, storageTr state.Trie) types.StateAccount { - codeBytes := make([]byte, 256) - _, err := r.Read(codeBytes) - require.NoError(t, err, "error reading random code bytes") - - codeHash := crypto.Keccak256Hash(codeBytes) - rawdb.WriteCode(serverDB.DiskDB(), codeHash, codeBytes) - account.CodeHash = codeHash[:] + if r.Intn(2) == 0 { + codeBytes := make([]byte, 256) + _, err := r.Read(codeBytes) + require.NoError(t, err, "error reading random code bytes") + + codeHash := crypto.Keccak256Hash(codeBytes) + rawdb.WriteCode(serverDB.DiskDB(), codeHash, codeBytes) + account.CodeHash = codeHash[:] + } // now create state trie FillStorageForAccount(t, r, 16, addr, storageTr) diff --git a/graft/evm/utils/utilstest/storage.go b/graft/evm/utils/utilstest/storage.go new file mode 100644 index 000000000000..41c1e3d96552 --- /dev/null +++ b/graft/evm/utils/utilstest/storage.go @@ -0,0 +1,69 @@ +// Copyright (C) 2019, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package utilstest + +import ( + "os" + "path/filepath" + "testing" + + "github.com/ava-labs/libevm/core/rawdb" + "github.com/ava-labs/libevm/ethdb" + "github.com/stretchr/testify/require" +) + +func CopyEthDB(t *testing.T, db ethdb.Database) ethdb.Database { + newDB := rawdb.NewMemoryDatabase() + iter := db.NewIterator(nil, nil) + defer iter.Release() + for iter.Next() { + require.NoError(t, newDB.Put(iter.Key(), iter.Value())) + } + + return newDB +} + +// CopyDir recursively copies all files and folders from a directory [src] to a +// new temporary directory and returns the path to the new directory. +func CopyDir(t *testing.T, src string) string { + t.Helper() + + if src == "" { + return "" + } + + dst := t.TempDir() + err := filepath.Walk(src, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // Calculate the relative path from src + relPath, err := filepath.Rel(src, path) + if err != nil { + return err + } + + // Skip the root directory itself + if relPath == "." { + return nil + } + + dstPath := filepath.Join(dst, relPath) + + if info.IsDir() { + return os.MkdirAll(dstPath, info.Mode().Perm()) + } + + data, err := os.ReadFile(path) + if err != nil { + return err + } + + return os.WriteFile(dstPath, data, info.Mode().Perm()) + }) + + require.NoError(t, err) + return dst +} diff --git a/graft/subnet-evm/core/blockchain_ext_test.go b/graft/subnet-evm/core/blockchain_ext_test.go index 714dd0be1da4..4b24b730fcd9 100644 --- a/graft/subnet-evm/core/blockchain_ext_test.go +++ b/graft/subnet-evm/core/blockchain_ext_test.go @@ -6,8 +6,6 @@ package core import ( "fmt" "math/big" - "os" - "path/filepath" "slices" "testing" @@ -20,6 +18,7 @@ import ( "github.com/holiman/uint256" "github.com/stretchr/testify/require" + "github.com/ava-labs/avalanchego/graft/evm/utils/utilstest" "github.com/ava-labs/avalanchego/graft/subnet-evm/commontype" "github.com/ava-labs/avalanchego/graft/subnet-evm/consensus/dummy" "github.com/ava-labs/avalanchego/graft/subnet-evm/params" @@ -127,63 +126,6 @@ var reexecTests = []ReexecTest{ }, } -func copyMemDB(db ethdb.Database) (ethdb.Database, error) { - newDB := rawdb.NewMemoryDatabase() - iter := db.NewIterator(nil, nil) - defer iter.Release() - for iter.Next() { - if err := newDB.Put(iter.Key(), iter.Value()); err != nil { - return nil, err - } - } - - return newDB, nil -} - -// copyDir recursively copies all files and folders from a directory [src] to a -// new temporary directory and returns the path to the new directory. -func copyDir(t *testing.T, src string) string { - t.Helper() - - if src == "" { - return "" - } - - dst := t.TempDir() - err := filepath.Walk(src, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - // Calculate the relative path from src - relPath, err := filepath.Rel(src, path) - if err != nil { - return err - } - - // Skip the root directory itself - if relPath == "." { - return nil - } - - dstPath := filepath.Join(dst, relPath) - - if info.IsDir() { - return os.MkdirAll(dstPath, info.Mode().Perm()) - } - - data, err := os.ReadFile(path) - if err != nil { - return err - } - - return os.WriteFile(dstPath, data, info.Mode().Perm()) - }) - - require.NoError(t, err) - return dst -} - // checkBlockChainState creates a new BlockChain instance and checks that exporting each block from // genesis to last accepted from the original instance yields the same last accepted block and state // root. @@ -229,9 +171,9 @@ func checkBlockChainState( require.NoErrorf(checkState(acceptedState), "Check state failed for newly generated blockchain") // Copy the database over to prevent any issues when re-using [originalDB] after this call. - originalDB, err = copyMemDB(originalDB) + originalDB = utilstest.CopyEthDB(t, originalDB) require.NoError(err) - newChainDataDir := copyDir(t, oldChainDataDir) + newChainDataDir := utilstest.CopyDir(t, oldChainDataDir) restartedChain, err := create(originalDB, gspec, lastAcceptedBlock.Hash(), newChainDataDir) require.NoError(err) defer restartedChain.Stop() @@ -1836,7 +1778,7 @@ func ReexecCorruptedStateTest(t *testing.T, create ReexecTestFunc) { blockchain.Stop() // Restart blockchain with existing state - newDir := copyDir(t, tempDir) // avoid file lock + newDir := utilstest.CopyDir(t, tempDir) // avoid file lock restartedBlockchain, err := create(chainDB, gspec, chain[1].Hash(), newDir, 4096) require.NoError(t, err) defer restartedBlockchain.Stop() diff --git a/graft/subnet-evm/core/chain_makers.go b/graft/subnet-evm/core/chain_makers.go index 3f246806fc56..9cd84fd7527a 100644 --- a/graft/subnet-evm/core/chain_makers.go +++ b/graft/subnet-evm/core/chain_makers.go @@ -34,7 +34,6 @@ import ( "github.com/ava-labs/avalanchego/graft/evm/constants" "github.com/ava-labs/avalanchego/graft/subnet-evm/commontype" "github.com/ava-labs/avalanchego/graft/subnet-evm/consensus" - "github.com/ava-labs/avalanchego/graft/subnet-evm/core/extstate" "github.com/ava-labs/avalanchego/graft/subnet-evm/params" "github.com/ava-labs/avalanchego/graft/subnet-evm/plugin/evm/customheader" "github.com/ava-labs/avalanchego/graft/subnet-evm/plugin/evm/customtypes" @@ -45,6 +44,7 @@ import ( "github.com/ava-labs/libevm/core/types" "github.com/ava-labs/libevm/core/vm" "github.com/ava-labs/libevm/ethdb" + "github.com/ava-labs/libevm/libevm/stateconf" "github.com/ava-labs/libevm/triedb" "github.com/holiman/uint256" ) @@ -275,6 +275,12 @@ func (b *BlockGen) SetOnBlockGenerated(onBlockGenerated func(*types.Block)) { // values. Inserting them into BlockChain requires use of FakePow or // a similar non-validating proof of work implementation. func GenerateChain(config *params.ChainConfig, parent *types.Block, engine consensus.Engine, db ethdb.Database, n int, gap uint64, gen func(int, *BlockGen)) ([]*types.Block, []types.Receipts, error) { + stateCache := state.NewDatabase(db) + defer stateCache.TrieDB().Close() + return GenerateChainFromStateCache(config, parent, engine, stateCache, n, gap, gen) +} + +func GenerateChainFromStateCache(config *params.ChainConfig, parent *types.Block, engine consensus.Engine, stateCache state.Database, n int, gap uint64, gen func(int, *BlockGen)) ([]*types.Block, []types.Receipts, error) { if config == nil { config = params.TestChainConfig } @@ -304,7 +310,8 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse } // Write state changes to db - root, err := statedb.Commit(b.header.Number.Uint64(), config.IsEIP158(b.header.Number)) + statedbOpts := stateconf.WithTrieDBUpdateOpts(stateconf.WithTrieDBUpdatePayload(block.ParentHash(), block.Hash())) + root, err := statedb.Commit(b.header.Number.Uint64(), config.IsEIP158(b.header.Number), statedbOpts) if err != nil { panic(fmt.Sprintf("state write error: %v", err)) } @@ -317,16 +324,12 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse return block, b.receipts, nil } - // Forcibly use hash-based state scheme for retaining all nodes in disk. - triedb := triedb.NewDatabase(db, triedb.HashDefaults) - defer triedb.Close() - for i := 0; i < n; i++ { - statedb, err := state.New(parent.Root(), extstate.NewDatabaseWithNodeDB(db, triedb), nil) + statedb, err := state.New(parent.Root(), stateCache, nil) if err != nil { return nil, nil, err } - block, receipts, err := genblock(i, parent, triedb, statedb) + block, receipts, err := genblock(i, parent, stateCache.TrieDB(), statedb) if err != nil { return nil, nil, err } diff --git a/graft/subnet-evm/network/network.go b/graft/subnet-evm/network/network.go index dd965eb402f1..935641663a0a 100644 --- a/graft/subnet-evm/network/network.go +++ b/graft/subnet-evm/network/network.go @@ -63,6 +63,7 @@ type SyncedNetworkClient interface { type Network interface { validators.Connector common.AppHandler + p2p.NodeSampler SyncedNetworkClient @@ -155,6 +156,24 @@ func NewNetwork( }, nil } +func (n *network) Sample(_ context.Context, limit int) []ids.NodeID { + if limit != 1 { + log.Warn("Sample called with limit > 1, but only 1 peer will be returned", "limit", limit) + } + + n.lock.Lock() + defer n.lock.Unlock() + node, ok, err := n.peers.GetAnyPeer(nil) + if err != nil { + log.Error("error getting peer from peer tracker", "error", err) + return nil + } + if !ok { + return nil + } + return []ids.NodeID{node} +} + // SendAppRequestAny synchronously sends request to an arbitrary peer with a // node version greater than or equal to minVersion. If minVersion is nil, // the request will be sent to any peer regardless of their version. diff --git a/graft/subnet-evm/plugin/evm/config/config.md b/graft/subnet-evm/plugin/evm/config/config.md index 1bc73774034d..1bd4dd6ec54e 100644 --- a/graft/subnet-evm/plugin/evm/config/config.md +++ b/graft/subnet-evm/plugin/evm/config/config.md @@ -199,7 +199,7 @@ Configuration is provided as a JSON object. All fields are optional unless other | Option | Type | Description | Default | |--------|------|-------------|---------| -| `metrics-expensive-enabled` | bool | Enable expensive debug-level metrics; this includes Firewood metrics | `true` | +| `metrics-expensive-enabled` | bool | Enable expensive debug-level metrics | `true` | ## Security and Access diff --git a/graft/subnet-evm/plugin/evm/syncervm_test.go b/graft/subnet-evm/plugin/evm/syncervm_test.go index d1c390410566..868f4b0f69b2 100644 --- a/graft/subnet-evm/plugin/evm/syncervm_test.go +++ b/graft/subnet-evm/plugin/evm/syncervm_test.go @@ -4,6 +4,7 @@ package evm import ( + "bytes" "context" "fmt" "math/big" @@ -19,11 +20,13 @@ import ( "github.com/ava-labs/libevm/log" "github.com/ava-labs/libevm/rlp" "github.com/ava-labs/libevm/trie" + "github.com/ava-labs/libevm/triedb" "github.com/stretchr/testify/require" "github.com/ava-labs/avalanchego/api/metrics" "github.com/ava-labs/avalanchego/database/prefixdb" "github.com/ava-labs/avalanchego/graft/evm/constants" + "github.com/ava-labs/avalanchego/graft/evm/firewood" "github.com/ava-labs/avalanchego/graft/evm/sync/client" "github.com/ava-labs/avalanchego/graft/evm/sync/engine" "github.com/ava-labs/avalanchego/graft/evm/sync/synctest" @@ -31,6 +34,7 @@ import ( "github.com/ava-labs/avalanchego/graft/subnet-evm/consensus/dummy" "github.com/ava-labs/avalanchego/graft/subnet-evm/core" "github.com/ava-labs/avalanchego/graft/subnet-evm/core/coretest" + "github.com/ava-labs/avalanchego/graft/subnet-evm/core/extstate" "github.com/ava-labs/avalanchego/graft/subnet-evm/params/paramstest" "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/snow" @@ -49,40 +53,50 @@ import ( ) func TestSkipStateSync(t *testing.T) { - rand.Seed(1) test := syncTest{ syncableInterval: 256, stateSyncMinBlocks: 300, // must be greater than [syncableInterval] to skip sync syncMode: block.StateSyncSkipped, } - vmSetup := createSyncServerAndClientVMs(t, test, engine.BlocksToFetch) - - testSyncerVM(t, vmSetup, test) + for _, scheme := range schemes { + t.Run(scheme, func(t *testing.T) { + test.stateScheme = scheme + vmSetup := createSyncServerAndClientVMs(t, test, engine.BlocksToFetch) + testSyncerVM(t, vmSetup, test) + }) + } } func TestStateSyncFromScratch(t *testing.T) { - rand.Seed(1) test := syncTest{ syncableInterval: 256, stateSyncMinBlocks: 50, // must be less than [syncableInterval] to perform sync syncMode: block.StateSyncStatic, } - vmSetup := createSyncServerAndClientVMs(t, test, engine.BlocksToFetch) - - testSyncerVM(t, vmSetup, test) + for _, scheme := range schemes { + t.Run(scheme, func(t *testing.T) { + test.stateScheme = scheme + vmSetup := createSyncServerAndClientVMs(t, test, engine.BlocksToFetch) + testSyncerVM(t, vmSetup, test) + }) + } } func TestStateSyncFromScratchExceedParent(t *testing.T) { - rand.Seed(1) numToGen := engine.BlocksToFetch + uint64(32) test := syncTest{ syncableInterval: numToGen, stateSyncMinBlocks: 50, // must be less than [syncableInterval] to perform sync syncMode: block.StateSyncStatic, } - vmSetup := createSyncServerAndClientVMs(t, test, int(numToGen)) - testSyncerVM(t, vmSetup, test) + for _, scheme := range schemes { + t.Run(scheme, func(t *testing.T) { + test.stateScheme = scheme + vmSetup := createSyncServerAndClientVMs(t, test, int(numToGen)) + testSyncerVM(t, vmSetup, test) + }) + } } func TestStateSyncToggleEnabledToDisabled(t *testing.T) { @@ -262,12 +276,12 @@ func createSyncServerAndClientVMs(t *testing.T, test syncTest, numBlocks int) *s // configure [serverVM] // Align commit intervals with the test's syncable interval so summaries are created // at the expected heights and Accept() does not skip. - serverConfigJSON := fmt.Sprintf(`{"commit-interval": %d, "state-sync-commit-interval": %d}`, - test.syncableInterval, test.syncableInterval, + serverConfigJSON := fmt.Sprintf(`"commit-interval": %d, "state-sync-commit-interval": %d, "state-history": %d`, + test.syncableInterval, test.syncableInterval, test.syncableInterval, ) serverVM := newVM(t, testVMConfig{ genesisJSON: toGenesisJSON(paramstest.ForkToChainConfig[upgradetest.Latest]), - configJSON: serverConfigJSON, + configJSON: getConfig(test.stateScheme, serverConfigJSON), }) t.Cleanup(func() { @@ -287,8 +301,24 @@ func createSyncServerAndClientVMs(t *testing.T, test syncTest, numBlocks int) *s }, nil) // make some accounts - r := rand.New(rand.NewSource(1)) - root, accounts := synctest.FillAccountsWithOverlappingStorage(t, r, serverVM.vm.Blockchain().StateCache(), types.EmptyRootHash, 1000, 16) + var ( + r = rand.New(rand.NewSource(1)) + currentRoot = serverVM.vm.blockChain.LastAcceptedBlock().Root() + root common.Hash + fundedAccounts, allAccounts map[*utilstest.Key]*types.StateAccount + ) + if test.stateScheme == customrawdb.FirewoodScheme { + tdb, ok := serverVM.vm.Blockchain().TrieDB().Backend().(*firewood.TrieDB) + require.True(ok) + tdb.SetHashAndHeight(common.Hash{}, 0) // must be set for FillAccountsWithStorageAndCode to work + } + root, allAccounts = synctest.FillAccountsWithStorageAndCode(t, r, serverVM.vm.Blockchain().StateCache(), currentRoot, 1000) + fundedAccounts = make(map[*utilstest.Key]*types.StateAccount) + for key, account := range allAccounts { + if len(account.CodeHash) == 0 || bytes.Equal(account.CodeHash, types.EmptyCodeHash[:]) { + fundedAccounts[key] = account + } + } // patch serverVM's lastAcceptedBlock to have the new root // and update the vm's state so the trie with accounts will @@ -304,12 +334,12 @@ func createSyncServerAndClientVMs(t *testing.T, test syncTest, numBlocks int) *s // initialise [syncerVM] with blank genesis state // Match the server's state-sync-commit-interval so parsed summaries are acceptable. stateSyncEnabledJSON := fmt.Sprintf( - `{"state-sync-enabled":true, "state-sync-min-blocks": %d, "tx-lookup-limit": %d, "state-sync-commit-interval": %d}`, + `"state-sync-enabled":true, "state-sync-min-blocks": %d, "tx-lookup-limit": %d, "state-sync-commit-interval": %d`, test.stateSyncMinBlocks, 4, test.syncableInterval, ) syncerVM := newVM(t, testVMConfig{ genesisJSON: toGenesisJSON(paramstest.ForkToChainConfig[upgradetest.Latest]), - configJSON: stateSyncEnabledJSON, + configJSON: getConfig(test.stateScheme, stateSyncEnabledJSON), isSyncing: true, }) @@ -361,7 +391,7 @@ func createSyncServerAndClientVMs(t *testing.T, test syncTest, numBlocks int) *s return &syncVMSetup{ serverVM: serverVM.vm, serverAppSender: serverVM.appSender, - fundedAccounts: accounts, + fundedAccounts: fundedAccounts, syncerVM: syncerVM.vm, syncerDB: syncerVM.db, shutdownOnceSyncerVM: shutdownOnceSyncerVM, @@ -398,6 +428,7 @@ type syncTest struct { stateSyncMinBlocks uint64 syncableInterval uint64 syncMode block.StateSyncMode + stateScheme string expectedErr error } @@ -568,11 +599,25 @@ func generateAndAcceptBlocks(t *testing.T, vm *VM, numBlocks int, gen func(int, accepted(block) } } - _, _, err := core.GenerateChain( + + lastAccepted := vm.Blockchain().LastAcceptedBlock() + c := vm.Blockchain().StateCache() + + // Firewood's state lives on disk, so we must copy the entire state into a new state cache, unlike HashDB. + if vm.Config().StateScheme == customrawdb.FirewoodScheme { + memdb := utilstest.CopyEthDB(t, vm.chaindb) + tdb := triedb.NewDatabase(memdb, &triedb.Config{ + DBOverride: firewood.DefaultConfig(utilstest.CopyDir(t, vm.Blockchain().CacheConfig().ChainDataDir)).BackendConstructor, + }) + tdb.Backend().(*firewood.TrieDB).SetHashAndHeight(lastAccepted.Hash(), lastAccepted.NumberU64()) + c = extstate.NewDatabaseWithNodeDB(memdb, tdb) + } + + _, _, err := core.GenerateChainFromStateCache( vm.chainConfig, - vm.blockChain.LastAcceptedBlock(), + lastAccepted, dummy.NewETHFaker(), - vm.chaindb, + c, numBlocks, 10, func(i int, g *core.BlockGen) { diff --git a/graft/subnet-evm/plugin/evm/vm.go b/graft/subnet-evm/plugin/evm/vm.go index 714b54ab4b0d..dbc1e28815d6 100644 --- a/graft/subnet-evm/plugin/evm/vm.go +++ b/graft/subnet-evm/plugin/evm/vm.go @@ -40,8 +40,10 @@ import ( "github.com/ava-labs/avalanchego/cache/metercacher" "github.com/ava-labs/avalanchego/codec" "github.com/ava-labs/avalanchego/database" + "github.com/ava-labs/avalanchego/database/merkle/firewood/syncer" "github.com/ava-labs/avalanchego/database/versiondb" "github.com/ava-labs/avalanchego/graft/evm/constants" + "github.com/ava-labs/avalanchego/graft/evm/firewood" "github.com/ava-labs/avalanchego/graft/evm/message" "github.com/ava-labs/avalanchego/graft/evm/rpc" "github.com/ava-labs/avalanchego/graft/evm/sync/client" @@ -149,7 +151,6 @@ var ( errVerifyGenesis = errors.New("failed to verify genesis") errFirewoodSnapshotCacheDisabled = errors.New("snapshot cache must be disabled for Firewood") errFirewoodOfflinePruningUnsupported = errors.New("offline pruning is not supported for Firewood") - errFirewoodStateSyncUnsupported = errors.New("state sync is not yet supported for Firewood") errFirewoodMissingTrieRepopulationUnsupported = errors.New("missing trie repopulation is not supported for Firewood") ) @@ -415,9 +416,6 @@ func (vm *VM) Initialize( if vm.config.OfflinePruning { return errFirewoodOfflinePruningUnsupported } - if vm.config.StateSyncEnabled { - return errFirewoodStateSyncUnsupported - } if vm.config.PopulateMissingTries != nil { return errFirewoodMissingTrieRepopulationUnsupported } @@ -643,32 +641,51 @@ func (vm *VM) initializeChain(lastAcceptedHash common.Hash, ethConfig ethconfig. // If state sync is disabled, this function will wipe any ongoing summary from // disk to ensure that we do not continue syncing from an invalid snapshot. func (vm *VM) initializeStateSync(lastAcceptedHeight uint64) error { - // Create standalone EVM TrieDB (read only) for serving leafs requests. - // We create a standalone TrieDB here, so that it has a standalone cache from the one - // used by the node when processing blocks. - evmTrieDB := triedb.NewDatabase( - vm.chaindb, - &triedb.Config{ - DBOverride: hashdb.Config{ - CleanCacheSize: vm.config.StateSyncServerTrieCache * units.MiB, - }.BackendConstructor, - }, - ) - - // register default leaf request handler for state trie + leafHandlers := make(LeafHandlers) + leafMetricsNames := make(map[message.NodeType]string) syncStats := handlerstats.GetOrRegisterHandlerStats(metrics.Enabled) - stateLeafRequestConfig := &extension.LeafRequestConfig{ - LeafType: message.StateTrieNode, - MetricName: "sync_state_trie_leaves", - Handler: handlers.NewLeafsRequestHandler(evmTrieDB, - message.StateTrieKeyLength, - vm.blockChain, vm.networkCodec, - syncStats, - ), - } - leafHandlers := make(LeafHandlers) - leafHandlers[stateLeafRequestConfig.LeafType] = stateLeafRequestConfig.Handler + switch scheme := vm.ethConfig.StateScheme; scheme { + case rawdb.HashScheme, "": + // Create standalone EVM TrieDB (read only) for serving leafs requests. + // We create a standalone TrieDB here, so that it has a standalone cache from the one + // used by the node when processing blocks. + evmTrieDB := triedb.NewDatabase( + vm.chaindb, + &triedb.Config{ + DBOverride: hashdb.Config{ + CleanCacheSize: vm.config.StateSyncServerTrieCache * units.MiB, + }.BackendConstructor, + }, + ) + // register default leaf request handler for state trie + stateLeafRequestConfig := &extension.LeafRequestConfig{ + LeafType: message.StateTrieNode, + MetricName: "sync_state_trie_leaves", + Handler: handlers.NewLeafsRequestHandler(evmTrieDB, + message.StateTrieKeyLength, + vm.blockChain, vm.networkCodec, + syncStats, + ), + } + leafHandlers[stateLeafRequestConfig.LeafType] = stateLeafRequestConfig.Handler + leafMetricsNames[stateLeafRequestConfig.LeafType] = stateLeafRequestConfig.MetricName + case customrawdb.FirewoodScheme: + tdb, ok := vm.eth.BlockChain().TrieDB().Backend().(*firewood.TrieDB) + if !ok { + return fmt.Errorf("expected a %T with %s scheme, got %T", tdb, customrawdb.FirewoodScheme, vm.eth.BlockChain().TrieDB().Backend()) + } + n := vm.Network.P2PNetwork() + if err := n.AddHandler(p2p.FirewoodRangeProofHandlerID, syncer.NewGetRangeProofHandler(tdb.Firewood)); err != nil { + return fmt.Errorf("adding firewood range proof handler: %w", err) + } + if err := n.AddHandler(p2p.FirewoodChangeProofHandlerID, syncer.NewGetChangeProofHandler(tdb.Firewood)); err != nil { + return fmt.Errorf("adding firewood change proof handler: %w", err) + } + default: + log.Warn("state sync is not supported for this scheme, no leaf handlers will be registered", "scheme", scheme) + return nil + } networkHandler := newNetworkHandler( vm.blockChain, @@ -694,14 +711,11 @@ func (vm *VM) initializeStateSync(lastAcceptedHeight uint64) error { } } - // Initialize the state sync client - leafMetricsNames := make(map[message.NodeType]string) - leafMetricsNames[stateLeafRequestConfig.LeafType] = stateLeafRequestConfig.MetricName - vm.Client = engine.NewClient(&engine.ClientConfig{ StateSyncDone: vm.stateSyncDone, Chain: newChainContextAdapter(vm.eth), State: vm.State, + SnowCtx: vm.ctx, Client: client.New( &client.Config{ Network: vm.Network, diff --git a/graft/subnet-evm/plugin/evm/vm_test.go b/graft/subnet-evm/plugin/evm/vm_test.go index 9fe92bae92af..1520e1de59d1 100644 --- a/graft/subnet-evm/plugin/evm/vm_test.go +++ b/graft/subnet-evm/plugin/evm/vm_test.go @@ -217,7 +217,7 @@ func getConfig(scheme, otherConfig string) string { if len(innerConfig) > 0 { innerConfig += ", " } - innerConfig += fmt.Sprintf(`"state-scheme": "%s", "snapshot-cache": 0, "pruning-enabled": true, "state-sync-enabled": false, "metrics-expensive-enabled": false`, customrawdb.FirewoodScheme) + innerConfig += fmt.Sprintf(`"state-scheme": "%s", "snapshot-cache": 0`, customrawdb.FirewoodScheme) } return fmt.Sprintf(`{%s}`, innerConfig) diff --git a/network/p2p/handler.go b/network/p2p/handler.go index a680299f0319..ddb0452764c9 100644 --- a/network/p2p/handler.go +++ b/network/p2p/handler.go @@ -26,6 +26,8 @@ const ( AtomicTxGossipHandlerID // SignatureRequestHandlerID is specified in ACP-118: https://github.com/avalanche-foundation/ACPs/tree/main/ACPs/118-warp-signature-request SignatureRequestHandlerID + FirewoodRangeProofHandlerID + FirewoodChangeProofHandlerID ) var (