Merge branch 'master' into darren/refactor/upgrade-cli-to-v3

Author: darrenvechain

.github/workflows/on-release.yaml

@@ -36,7 +36,7 @@ jobs:
       - name: Validate API Version
         run: |
           thor_version=$(cat cmd/thor/VERSION)
-          api_version=$(grep -o 'version: [0-9.]*' api/doc/thor.yaml | awk '{print $2}')
+          api_version=$(grep -o 'version: [0-9][0-9.]*' api/doc/thor.yaml | awk '{print $2}')
           if [ "$thor_version" != "$api_version" ]; then
             echo "API spec version ($api_version) does not match Thor version ($thor_version)"
             exit 1

chain/chain.go

@@ -24,12 +24,6 @@ import (
 	"github.com/vechain/thor/v2/tx"
 )
 
-const (
-	// IndexTrieName is the name of index trie.
-	// The index tire is used to store mappings from block number to block id, and tx id to tx meta.
-	IndexTrieName = "i"
-)
-
 type storageTxMeta struct {
 	Index    uint64
 	Reverted bool
@@ -57,6 +51,7 @@ type Chain struct {
 
 func newChain(repo *Repository, headID thor.Bytes32) *Chain {
 	var (
+		// The index tire is used to store mappings from block number to block id, and tx id to tx meta.
 		indexTrie *muxdb.Trie
 		initErr   error
 	)
@@ -67,7 +62,7 @@ func newChain(repo *Repository, headID thor.Bytes32) *Chain {
 		func() (*muxdb.Trie, error) {
 			if indexTrie == nil && initErr == nil {
 				if summary, err := repo.GetBlockSummary(headID); err == nil {
-					indexTrie = repo.db.NewTrie(IndexTrieName, summary.IndexRoot())
+					indexTrie = repo.db.NewTrie(muxdb.IndexTrieName, summary.IndexRoot())
 				} else {
 					initErr = errors.Wrap(err, fmt.Sprintf("lazy init chain, head=%v", headID))
 				}
@@ -373,7 +368,7 @@ func (r *Repository) NewChain(headID thor.Bytes32) *Chain {
 }
 
 func (r *Repository) indexBlock(parentRoot trie.Root, newBlockID thor.Bytes32, newConflicts uint32) error {
-	t := r.db.NewTrie(IndexTrieName, parentRoot)
+	t := r.db.NewTrie(muxdb.IndexTrieName, parentRoot)
 	// map block number to block ID
 	if err := t.Update(newBlockID[:4], newBlockID[:], nil); err != nil {
 		return err

cmd/thor/main.go

@@ -320,7 +320,7 @@ func defaultAction(_ context.Context, ctx *cli.Command) error {
 	defer p2pCommunicator.Stop()
 
 	if !ctx.Bool(disablePrunerFlag.Name) {
-		pruner := pruner.New(mainDB, repo, bftEngine)
+		pruner := pruner.New(mainDB, repo, bftEngine, *forkConfig)
 		defer func() { log.Info("stopping pruner..."); pruner.Stop() }()
 	}
 
@@ -510,7 +510,7 @@ func soloAction(_ context.Context, ctx *cli.Command) error {
 	printStartupMessage2(gene, apiURL, "", metricsURL, adminURL, isDevnet)
 
 	if !ctx.Bool(disablePrunerFlag.Name) {
-		pruner := pruner.New(mainDB, repo, bftMockedEngine)
+		pruner := pruner.New(mainDB, repo, bftMockedEngine, *forkConfig)
 		defer func() { log.Info("stopping pruner..."); pruner.Stop() }()
 	}
 

cmd/thor/pruner/pruner.go

@@ -8,6 +8,7 @@ package pruner
 import (
 	"context"
 	"fmt"
+	"math"
 	"sync"
 	"time"
 
@@ -40,17 +41,19 @@ type Pruner struct {
 	commiter bft.Committer
 	cancel   func()
 	goes     sync.WaitGroup
+	fc       *thor.ForkConfig
 }
 
 // New creates and starts the pruner.
-func New(db *muxdb.MuxDB, repo *chain.Repository, commiter bft.Committer) *Pruner {
+func New(db *muxdb.MuxDB, repo *chain.Repository, commiter bft.Committer, fc thor.ForkConfig) *Pruner {
 	ctx, cancel := context.WithCancel(context.Background())
 	o := &Pruner{
 		db:       db,
 		repo:     repo,
 		ctx:      ctx,
 		commiter: commiter,
 		cancel:   cancel,
+		fc:       &fc,
 	}
 	o.goes.Go(func() {
 		if err := o.loop(); err != nil {
@@ -90,9 +93,11 @@ func (p *Pruner) loop() error {
 		// select target
 		target := status.Base + period
 
-		targetChain, err := p.awaitUntilFinalized(target + thor.MaxStateHistory)
+		// adding thor.MaxStateHistory here since we need to ensure that defined range of history is required
+		// to be kept for EVM accessibility. It's defined in thor/params.go(thor.MaxStateHistory ~7 days).
+		targetChain, err := p.awaitUntilPrunable(target + thor.MaxStateHistory)
 		if err != nil {
-			return errors.Wrap(err, "awaitUntilFinalized")
+			return errors.Wrap(err, "awaitUntilPrunable")
 		}
 		startTime := time.Now().UnixNano()
 
@@ -153,15 +158,15 @@ func (p *Pruner) checkpointTries(targetChain *chain.Chain, base, target uint32)
 	}
 
 	// checkpoint index trie
-	indexTrie := p.db.NewTrie(chain.IndexTrieName, summary.IndexRoot())
+	indexTrie := p.db.NewTrie(muxdb.IndexTrieName, summary.IndexRoot())
 	indexTrie.SetNoFillCache(true)
 
 	if err := indexTrie.Checkpoint(p.ctx, base, nil); err != nil {
 		return err
 	}
 
 	// checkpoint account trie
-	accTrie := p.db.NewTrie(state.AccountTrieName, summary.Root())
+	accTrie := p.db.NewTrie(muxdb.AccountTrieName, summary.Root())
 	accTrie.SetNoFillCache(true)
 
 	var sTries []*muxdb.Trie
@@ -195,9 +200,13 @@ func (p *Pruner) pruneTries(targetChain *chain.Chain, base, target uint32) error
 	return nil
 }
 
-// awaitUntilFinalized waits until the target block number becomes almost final(steady),
-// and returns the steady chain.
-func (p *Pruner) awaitUntilFinalized(target uint32) (*chain.Chain, error) {
+// awaitUntilPrunable waits until the target block number becomes prunable,and returns the prunable chain.
+// Before the finality hard fork, it's awaitUntilSteady. After the finality hard fork, it's awaitUntilFinalized.
+func (p *Pruner) awaitUntilPrunable(target uint32) (*chain.Chain, error) {
+	if p.fc.FINALITY > target {
+		return p.awaitUntilSteady(target)
+	}
+
 	for {
 		finalizedID := p.commiter.Finalized()
 		finalizedNum := block.Number(finalizedID)
@@ -217,3 +226,49 @@ func (p *Pruner) awaitUntilFinalized(target uint32) (*chain.Chain, error) {
 		}
 	}
 }
+
+func (p *Pruner) awaitUntilSteady(target uint32) (*chain.Chain, error) {
+	const windowSize = 100000
+
+	backoff := uint32(0)
+	for {
+		best := p.repo.BestBlockSummary()
+		bestNum := best.Header.Number()
+		if bestNum > target+backoff {
+			var meanScore float64
+			if bestNum > windowSize {
+				baseNum := bestNum - windowSize
+				baseHeader, err := p.repo.NewChain(best.Header.ID()).GetBlockHeader(baseNum)
+				if err != nil {
+					return nil, err
+				}
+				meanScore = math.Round(float64(best.Header.TotalScore()-baseHeader.TotalScore()) / float64(windowSize))
+			} else {
+				meanScore = math.Round(float64(best.Header.TotalScore()) / float64(bestNum))
+			}
+			set := make(map[thor.Address]struct{})
+			// reverse iterate the chain and collect signers.
+			for i, prev := 0, best.Header; i < int(meanScore*3) && prev.Number() >= target; i++ {
+				signer, _ := prev.Signer()
+				set[signer] = struct{}{}
+				if len(set) >= int(math.Round((meanScore+1)/2)) {
+					// got enough unique signers
+					steadyID := prev.ID()
+					return p.repo.NewChain(steadyID), nil
+				}
+				parent, err := p.repo.GetBlockSummary(prev.ParentID())
+				if err != nil {
+					return nil, err
+				}
+				prev = parent.Header
+			}
+			backoff += uint32(meanScore)
+		} else {
+			select {
+			case <-p.ctx.Done():
+				return nil, p.ctx.Err()
+			case <-time.After(time.Second):
+			}
+		}
+	}
+}

cmd/thor/pruner/pruner_test.go

@@ -13,10 +13,12 @@ import (
 	"math/big"
 	"os"
 	"path/filepath"
+	"slices"
 	"testing"
 	"time"
 
 	"github.com/vechain/thor/v2/bft"
+	"github.com/vechain/thor/v2/builtin"
 	"github.com/vechain/thor/v2/test/testchain"
 
 	"github.com/ethereum/go-ethereum/crypto"
@@ -108,12 +110,12 @@ func TestStatus(t *testing.T) {
 func TestNewPruner(t *testing.T) {
 	db := muxdb.NewMem()
 	stater := state.NewStater(db)
-	gene, _ := genesis.NewDevnet()
+	gene, fc := genesis.NewDevnet()
 	b0, _, _, _ := gene.Build(stater)
 	repo, _ := chain.NewRepository(db, b0)
 
 	bftMockedEngine := bft.NewMockedEngine(repo.GenesisBlock().Header().ID())
-	pr := New(db, repo, bftMockedEngine)
+	pr := New(db, repo, bftMockedEngine, *fc)
 	pr.Stop()
 }
 
@@ -188,7 +190,7 @@ func (tc *testCommitter) ShouldVote(parentID thor.Bytes32) (bool, error) {
 func TestWaitUntil(t *testing.T) {
 	db := muxdb.NewMem()
 	stater := state.NewStater(db)
-	gene, _ := genesis.NewDevnet()
+	gene, fc := genesis.NewDevnet()
 	b0, _, _, _ := gene.Build(stater)
 	repo, _ := chain.NewRepository(db, b0)
 	devAccounts := genesis.DevAccounts()
@@ -201,6 +203,7 @@ func TestWaitUntil(t *testing.T) {
 		ctx:      ctx,
 		commiter: testCommiter,
 		cancel:   cancel,
+		fc:       fc,
 	}
 
 	parentID := b0.Header().ID()
@@ -231,7 +234,7 @@ func TestWaitUntil(t *testing.T) {
 
 	cancel()
 	// Use a target that doesn't exist yet to force waiting (where cancellation is checked)
-	_, err = pruner.awaitUntilFinalized(200000) // Target beyond current best
+	_, err = pruner.awaitUntilPrunable(200000) // Target beyond current best
 	assert.NotNil(t, err)
 	assert.Equal(t, context.Canceled, err)
 
@@ -250,7 +253,7 @@ func TestWaitUntil(t *testing.T) {
 	pruner.ctx = ctx
 	pruner.cancel = cancel
 
-	chain, err := pruner.awaitUntilFinalized(100000)
+	chain, err := pruner.awaitUntilPrunable(100000)
 	assert.Nil(t, err)
 
 	assert.True(t, block.Number(chain.HeadID()) >= 10000)
@@ -344,3 +347,107 @@ func TestGetAfterPrune(t *testing.T) {
 	assert.Error(t, err)
 	assert.Contains(t, err.Error(), "missing trie node")
 }
+
+func TestGetStorageRandomlyTouchedAfterPrune(t *testing.T) {
+	type testcase struct {
+		name           string
+		blocks         []int
+		expectedEnergy uint64
+	}
+
+	cases := []testcase{
+		{
+			name:           "touch storage every block",
+			blocks:         []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
+			expectedEnergy: uint64(10),
+		},
+		{
+			name:           "touch only once at 9",
+			blocks:         []int{9},
+			expectedEnergy: uint64(1),
+		},
+		{
+			name:           "touch only once at 10",
+			blocks:         []int{10},
+			expectedEnergy: uint64(1),
+		},
+		{
+			name:           "touch randomly before pruning point case 1",
+			blocks:         []int{5, 7, 8},
+			expectedEnergy: uint64(3),
+		},
+		{
+			name:           "touch randomly before pruning point case 2",
+			blocks:         []int{5, 7},
+			expectedEnergy: uint64(2),
+		},
+		{
+			name:           "touch randomly before pruning point case 3",
+			blocks:         []int{3, 6},
+			expectedEnergy: uint64(2),
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			chain, err := testchain.NewDefault()
+			assert.NoError(t, err)
+
+			accounts := genesis.DevAccounts()
+			to := thor.BytesToAddress([]byte("to"))
+
+			// prepare energy transfer data
+			transferMethod, ok := builtin.Energy.ABI.MethodByName("transfer")
+			assert.True(t, ok)
+			transferData, err := transferMethod.EncodeInput(to, big.NewInt(1))
+			assert.NoError(t, err)
+
+			for i := range 20 {
+				clauses := []*tx.Clause{tx.NewClause(&to).WithValue(big.NewInt(1))}
+				if contains(tc.blocks, i+1) {
+					// touch energy storage by transferring 1 wei VTHO
+					clauses = append(clauses, tx.NewClause(&builtin.Energy.Address).WithData(transferData))
+				}
+
+				err = chain.MintClauses(accounts[0], clauses)
+				assert.NoError(t, err)
+			}
+
+			pruner := Pruner{
+				repo: chain.Repo(),
+				db:   chain.Database(),
+			}
+			// iterate best chain to 20
+
+			// prune [0, 10)
+			blk10, err := chain.Repo().NewBestChain().GetBlockSummary(10)
+			if err != nil {
+				t.Fatalf("failed to get block 10: %v", err)
+			}
+			err = pruner.pruneTries(chain.Repo().NewChain(blk10.Header.ID()), 0, blk10.Header.Number())
+			assert.NoError(t, err)
+
+			st := chain.State()
+			balance, err := st.GetEnergy(to, chain.Repo().BestBlockSummary().Header.Timestamp(), math.MaxUint64)
+			assert.NoError(t, err)
+			assert.Equal(t, tc.expectedEnergy, balance.Uint64())
+
+			st = state.NewStater(chain.Database()).NewState(blk10.Root())
+			_, err = st.GetEnergy(to, blk10.Header.Timestamp(), math.MaxUint64)
+			assert.NoError(t, err)
+			assert.Equal(t, tc.expectedEnergy, balance.Uint64())
+
+			sum, err := chain.Repo().NewBestChain().GetBlockSummary(9)
+			assert.NoError(t, err)
+
+			st = state.NewStater(chain.Database()).NewState(sum.Root())
+			_, err = st.GetEnergy(to, sum.Header.Timestamp(), math.MaxUint64)
+			assert.Error(t, err)
+			assert.Contains(t, err.Error(), "missing trie node")
+		})
+	}
+}
+
+func contains(slice []int, val int) bool {
+	return slices.Contains(slice, val)
+}