[WIP] all: track state changes in state db 27349

core/state/database.go

@@ -137,6 +137,16 @@ func NewDatabaseWithConfig(db ethdb.Database, config *trie.Config) Database {
 	}
 }
 
+// NewDatabaseWithNodeDB creates a state database with an already initialized node database.
+func NewDatabaseWithNodeDB(db ethdb.Database, triedb *trie.Database) Database {
+	return &cachingDB{
+		db:            triedb,
+		disk:          db,
+		codeCache:     lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize),
+		codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize),
+	}
+}
+
 type cachingDB struct {
 	db            *trie.Database
 	disk          ethdb.KeyValueStore

core/state/dump.go

@@ -166,7 +166,7 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey []
 		} else {
 			address = &addr
 		}
-		obj := newObject(s, addr, data)
+		obj := newObject(s, addr, &data)
 		if !conf.SkipCode {
 			account.Code = obj.Code(s.db)
 		}

core/state/journal.go

@@ -93,6 +93,12 @@ type (
 		account      *common.Address
 		prev         *stateObject
 		prevdestruct bool
+		prevAccount  []byte
+		prevStorage  map[common.Hash][]byte
+
+		prevAccountOriginExist bool
+		prevAccountOrigin      []byte
+		prevStorageOrigin      map[common.Hash][]byte
 	}
 	selfDestructChange struct {
 		account     *common.Address
@@ -160,6 +166,18 @@ func (ch resetObjectChange) revert(s *StateDB) {
 	if !ch.prevdestruct {
 		delete(s.stateObjectsDestruct, ch.prev.address)
 	}
+	if ch.prevAccount != nil {
+		s.accounts[ch.prev.addrHash] = ch.prevAccount
+	}
+	if ch.prevStorage != nil {
+		s.storages[ch.prev.addrHash] = ch.prevStorage
+	}
+	if ch.prevAccountOriginExist {
+		s.accountsOrigin[ch.prev.addrHash] = ch.prevAccountOrigin
+	}
+	if ch.prevStorageOrigin != nil {
+		s.storagesOrigin[ch.prev.addrHash] = ch.prevStorageOrigin
+	}
 }
 
 func (ch resetObjectChange) dirtied() *common.Address {

core/state/metrics.go

@@ -19,10 +19,19 @@ package state
 import "github.com/XinFinOrg/XDPoSChain/metrics"
 
 var (
-	accountUpdatedMeter        = metrics.NewRegisteredMeter("state/update/account", nil)
-	storageUpdatedMeter        = metrics.NewRegisteredMeter("state/update/storage", nil)
-	accountDeletedMeter        = metrics.NewRegisteredMeter("state/delete/account", nil)
-	storageDeletedMeter        = metrics.NewRegisteredMeter("state/delete/storage", nil)
-	accountTrieCommittedMeter  = metrics.NewRegisteredMeter("state/commit/accountnodes", nil)
-	storageTriesCommittedMeter = metrics.NewRegisteredMeter("state/commit/storagenodes", nil)
+	accountUpdatedMeter      = metrics.NewRegisteredMeter("state/update/account", nil)
+	storageUpdatedMeter      = metrics.NewRegisteredMeter("state/update/storage", nil)
+	accountDeletedMeter      = metrics.NewRegisteredMeter("state/delete/account", nil)
+	storageDeletedMeter      = metrics.NewRegisteredMeter("state/delete/storage", nil)
+	accountTrieUpdatedMeter  = metrics.NewRegisteredMeter("state/update/accountnodes", nil)
+	storageTriesUpdatedMeter = metrics.NewRegisteredMeter("state/update/storagenodes", nil)
+	accountTrieDeletedMeter  = metrics.NewRegisteredMeter("state/delete/accountnodes", nil)
+	storageTriesDeletedMeter = metrics.NewRegisteredMeter("state/delete/storagenodes", nil)
+
+	slotDeletionMaxCount = metrics.NewRegisteredGauge("state/delete/storage/max/slot", nil)
+	slotDeletionMaxSize  = metrics.NewRegisteredGauge("state/delete/storage/max/size", nil)
+	slotDeletionTimer    = metrics.NewRegisteredResettingTimer("state/delete/storage/timer", nil)
+	slotDeletionCount    = metrics.NewRegisteredMeter("state/delete/storage/slot", nil)
+	slotDeletionSize     = metrics.NewRegisteredMeter("state/delete/storage/size", nil)
+	slotDeletionSkip     = metrics.NewRegisteredGauge("state/delete/storage/skip", nil)
 )

core/state/state_object.go

@@ -59,14 +59,15 @@ func (s Storage) Copy() Storage {
 // stateObject represents an Ethereum account which is being modified.
 //
 // The usage pattern is as follows:
-// First you need to obtain a state object.
-// Account values can be accessed and modified through the object.
-// Finally, call commitTrie to write the modified storage trie into a database.
+// - First you need to obtain a state object.
+// - Account values as well as storages can be accessed and modified through the object.
+// - Finally, call commit to return the changes of storage trie and update account data.
 type stateObject struct {
 	db       *StateDB
-	address  common.Address     // address of ethereum account
-	addrHash common.Hash        // hash of ethereum address of the account
-	data     types.StateAccount // Account data with all mutations applied in the scope of block
+	address  common.Address      // address of ethereum account
+	addrHash common.Hash         // hash of ethereum address of the account
+	origin   *types.StateAccount // Account original data without any change applied, nil means it was not existent
+	data     types.StateAccount  // Account data with all mutations applied in the scope of block
 
 	// DB error.
 	// State objects are used by the consensus core and VM which are
@@ -79,9 +80,9 @@ type stateObject struct {
 	trie Trie // storage trie, which becomes non-nil on first access
 	code Code // contract bytecode, which gets set when code is loaded
 
-	originStorage  Storage // Storage cache of original entries to dedup rewrites, reset for every transaction
+	originStorage  Storage // Storage cache of original entries to dedup rewrites
 	pendingStorage Storage // Storage entries that need to be flushed to disk, at the end of an entire block
-	dirtyStorage   Storage // Storage entries that need to be flushed to disk
+	dirtyStorage   Storage // Storage entries that need to be flushed to disk, reset for every transaction
 
 	// Cache flags.
 	dirtyCode bool // true if the code was updated
@@ -105,21 +106,17 @@ func (s *stateObject) empty() bool {
 }
 
 // newObject creates a state object.
-func newObject(db *StateDB, address common.Address, data types.StateAccount) *stateObject {
-	if data.Balance == nil {
-		data.Balance = new(big.Int)
-	}
-	if data.CodeHash == nil {
-		data.CodeHash = types.EmptyCodeHash.Bytes()
-	}
-	if data.Root == (common.Hash{}) {
-		data.Root = types.EmptyRootHash
+func newObject(db *StateDB, address common.Address, acct *types.StateAccount) *stateObject {
+	origin := acct
+	if acct == nil {
+		acct = types.NewEmptyStateAccount()
 	}
 	return &stateObject{
 		db:             db,
 		address:        address,
 		addrHash:       crypto.Keccak256Hash(address[:]),
-		data:           data,
+		origin:         origin,
+		data:           *acct,
 		originStorage:  make(Storage),
 		pendingStorage: make(Storage),
 		dirtyStorage:   make(Storage),
@@ -266,34 +263,74 @@ func (s *stateObject) updateTrie(db Database) (Trie, error) {
 	}
 	// Track the amount of time wasted on updating the storage trie
 	defer func(start time.Time) { s.db.StorageUpdates += time.Since(start) }(time.Now())
+	// The snapshot storage map for the object
+	var (
+		storage map[common.Hash][]byte
+		origin  map[common.Hash][]byte
+		hasher  = s.db.hasher
+	)
 	tr, err := s.getTrie(db)
 	if err != nil {
 		s.setError(err)
 		return nil, err
 	}
 	// Insert all the pending updates into the trie
+	usedStorage := make([][]byte, 0, len(s.pendingStorage))
 	for key, value := range s.pendingStorage {
 		// Skip noop changes, persist actual changes
 		if value == s.originStorage[key] {
 			continue
 		}
+		prev := s.originStorage[key]
 		s.originStorage[key] = value
 
-		if (value == common.Hash{}) {
+		// rlp-encoded value to be used by the snapshot
+		var snapshotVal []byte
+		if value == (common.Hash{}) {
 			if err := tr.TryDelete(key[:]); err != nil {
 				s.setError(err)
 				return nil, err
 			}
 			s.db.StorageDeleted += 1
 		} else {
+			trimmedVal := common.TrimLeftZeroes(value[:])
 			// Encoding []byte cannot fail, ok to ignore the error.
-			v, _ := rlp.EncodeToBytes(common.TrimLeftZeroes(value[:]))
-			if err := tr.TryUpdate(key[:], v); err != nil {
+			snapshotVal, _ := rlp.EncodeToBytes(trimmedVal)
+			if err := tr.TryUpdate(key[:], snapshotVal); err != nil {
 				s.setError(err)
 				return nil, err
 			}
 			s.db.StorageUpdated += 1
 		}
+		// Cache the mutated storage slots until commit
+		if storage == nil {
+			if storage = s.db.storages[s.addrHash]; storage == nil {
+				storage = make(map[common.Hash][]byte)
+				s.db.storages[s.addrHash] = storage
+			}
+		}
+		khash := crypto.HashData(hasher, key[:])
+		storage[khash] = snapshotVal // snapshotVal will be nil if it's deleted
+
+		// Cache the original value of mutated storage slots
+		if origin == nil {
+			if origin = s.db.storagesOrigin[s.addrHash]; origin == nil {
+				origin = make(map[common.Hash][]byte)
+				s.db.storagesOrigin[s.addrHash] = origin
+			}
+		}
+		// Track the original value of slot only if it's mutated first time
+		if _, ok := origin[khash]; !ok {
+			if prev == (common.Hash{}) {
+				origin[khash] = nil // nil if it was not present previously
+			} else {
+				// Encoding []byte cannot fail, ok to ignore the error.
+				b, _ := rlp.EncodeToBytes(common.TrimLeftZeroes(prev[:]))
+				origin[khash] = b
+			}
+		}
+		// Cache the items for preloading
+		usedStorage = append(usedStorage, common.CopyBytes(key[:])) // Copy needed for closure
 	}
 	if len(s.pendingStorage) > 0 {
 		s.pendingStorage = make(Storage)
@@ -318,9 +355,8 @@ func (s *stateObject) updateRoot(db Database) {
 	s.data.Root = tr.Hash()
 }
 
-// CommitTrie the storage trie of the object to dwb.
-// This updates the trie root.
-func (s *stateObject) commitTrie(db Database) (*trie.NodeSet, error) {
+// commit returns the changes made in storage trie and updates the account data.
+func (s *stateObject) commit(db Database) (*trie.NodeSet, error) {
 	// If nothing changed, don't bother with hashing anything
 	tr, err := s.updateTrie(db)
 	if err != nil {
@@ -331,6 +367,7 @@ func (s *stateObject) commitTrie(db Database) (*trie.NodeSet, error) {
 	}
 	// If nothing changed, don't bother with hashing anything
 	if tr == nil {
+		s.origin = s.data.Copy()
 		return nil, nil
 	}
 	// Track the amount of time wasted on committing the storage trie
@@ -339,6 +376,9 @@ func (s *stateObject) commitTrie(db Database) (*trie.NodeSet, error) {
 	if err == nil {
 		s.data.Root = root
 	}
+
+	// Update original account data after commit
+	s.origin = s.data.Copy()
 	return nodes, err
 }
 
@@ -381,18 +421,24 @@ func (s *stateObject) setBalance(amount *big.Int) {
 }
 
 func (s *stateObject) deepCopy(db *StateDB) *stateObject {
-	stateObject := newObject(db, s.address, s.data)
+	obj := &stateObject{
+		db:       db,
+		address:  s.address,
+		addrHash: s.addrHash,
+		origin:   s.origin,
+		data:     s.data,
+	}
 	if s.trie != nil {
-		stateObject.trie = db.db.CopyTrie(s.trie)
-	}
-	stateObject.code = s.code
-	stateObject.dirtyStorage = s.dirtyStorage.Copy()
-	stateObject.originStorage = s.originStorage.Copy()
-	stateObject.pendingStorage = s.pendingStorage.Copy()
-	stateObject.selfDestructed = s.selfDestructed
-	stateObject.dirtyCode = s.dirtyCode
-	stateObject.deleted = s.deleted
-	return stateObject
+		obj.trie = db.db.CopyTrie(s.trie)
+	}
+	obj.code = s.code
+	obj.dirtyStorage = s.dirtyStorage.Copy()
+	obj.originStorage = s.originStorage.Copy()
+	obj.pendingStorage = s.pendingStorage.Copy()
+	obj.selfDestructed = s.selfDestructed
+	obj.dirtyCode = s.dirtyCode
+	obj.deleted = s.deleted
+	return obj
 }
 
 //

core/state/state_test.go

@@ -31,21 +31,21 @@ import (
 	"github.com/XinFinOrg/XDPoSChain/trie"
 )
 
-type stateTest struct {
+type stateEnv struct {
 	db    ethdb.Database
 	state *StateDB
 }
 
-func newStateTest() *stateTest {
+func newStateEnv() *stateEnv {
 	db := rawdb.NewMemoryDatabase()
 	sdb, _ := New(common.Hash{}, NewDatabase(db))
-	return &stateTest{db: db, state: sdb}
+	return &stateEnv{db: db, state: sdb}
 }
 
 func TestDump(t *testing.T) {
 	db := rawdb.NewMemoryDatabase()
 	sdb, _ := New(common.Hash{}, NewDatabaseWithConfig(db, &trie.Config{Preimages: true}))
-	s := &stateTest{db: db, state: sdb}
+	s := &stateEnv{db: db, state: sdb}
 
 	// generate a few entries
 	obj1 := s.state.GetOrNewStateObject(common.BytesToAddress([]byte{0x01}))
@@ -97,7 +97,7 @@ func TestDump(t *testing.T) {
 func TestIterativeDump(t *testing.T) {
 	db := rawdb.NewMemoryDatabase()
 	sdb, _ := New(types.EmptyRootHash, NewDatabaseWithConfig(db, &trie.Config{Preimages: true}))
-	s := &stateTest{db: db, state: sdb}
+	s := &stateEnv{db: db, state: sdb}
 
 	// generate a few entries
 	obj1 := s.state.GetOrNewStateObject(common.BytesToAddress([]byte{0x01}))
@@ -130,7 +130,7 @@ func TestIterativeDump(t *testing.T) {
 }
 
 func TestNull(t *testing.T) {
-	s := newStateTest()
+	s := newStateEnv()
 	address := common.HexToAddress("0x823140710bf13990e4500136726d8b55")
 	s.state.CreateAccount(address)
 	//value := common.FromHex("0x823140710bf13990e4500136726d8b55")
@@ -152,7 +152,7 @@ func TestSnapshot(t *testing.T) {
 	var storageaddr common.Hash
 	data1 := common.BytesToHash([]byte{42})
 	data2 := common.BytesToHash([]byte{43})
-	s := newStateTest()
+	s := newStateEnv()
 
 	// snapshot the genesis state
 	genesis := s.state.Snapshot()
@@ -183,7 +183,7 @@ func TestSnapshot(t *testing.T) {
 }
 
 func TestSnapshotEmpty(t *testing.T) {
-	s := newStateTest()
+	s := newStateEnv()
 	s.state.RevertToSnapshot(s.state.Snapshot())
 }