Merge pull request #6746 from onflow/leo/cmd-add-verify-execution-result

[Util] Add verify execution result cmd

Author: zhangchiqing

cmd/scaffold.go

@@ -33,8 +33,7 @@ import (
 	"github.com/onflow/flow-go/cmd/build"
 	"github.com/onflow/flow-go/config"
 	"github.com/onflow/flow-go/consensus/hotstuff/persister"
-	"github.com/onflow/flow-go/fvm"
-	"github.com/onflow/flow-go/fvm/environment"
+	"github.com/onflow/flow-go/fvm/initialize"
 	"github.com/onflow/flow-go/model/flow"
 	"github.com/onflow/flow-go/model/flow/filter"
 	"github.com/onflow/flow-go/module"
@@ -1522,32 +1521,9 @@ func (fnb *FlowNodeBuilder) initLocal() error {
 }
 
 func (fnb *FlowNodeBuilder) initFvmOptions() {
-	blockFinder := environment.NewBlockFinder(fnb.Storage.Headers)
-	vmOpts := []fvm.Option{
-		fvm.WithChain(fnb.RootChainID.Chain()),
-		fvm.WithBlocks(blockFinder),
-		fvm.WithAccountStorageLimit(true),
-	}
-	switch fnb.RootChainID {
-	case flow.Testnet,
-		flow.Sandboxnet,
-		flow.Previewnet,
-		flow.Mainnet:
-		vmOpts = append(vmOpts,
-			fvm.WithTransactionFeesEnabled(true),
-		)
-	}
-	switch fnb.RootChainID {
-	case flow.Testnet,
-		flow.Sandboxnet,
-		flow.Previewnet,
-		flow.Localnet,
-		flow.Benchnet:
-		vmOpts = append(vmOpts,
-			fvm.WithContractDeploymentRestricted(false),
-		)
-	}
-	fnb.FvmOptions = vmOpts
+	fnb.FvmOptions = initialize.InitFvmOptions(
+		fnb.RootChainID, fnb.Storage.Headers,
+	)
 }
 
 // handleModules initializes the given module.

cmd/util/cmd/root.go

@@ -41,6 +41,7 @@ import (
 	"github.com/onflow/flow-go/cmd/util/cmd/snapshot"
 	system_addresses "github.com/onflow/flow-go/cmd/util/cmd/system-addresses"
 	truncate_database "github.com/onflow/flow-go/cmd/util/cmd/truncate-database"
+	verify_execution_result "github.com/onflow/flow-go/cmd/util/cmd/verify_execution_result"
 	"github.com/onflow/flow-go/cmd/util/cmd/version"
 	"github.com/onflow/flow-go/module/profiler"
 )
@@ -126,6 +127,7 @@ func addCommands() {
 	rootCmd.AddCommand(debug_script.Cmd)
 	rootCmd.AddCommand(generate_authorization_fixes.Cmd)
 	rootCmd.AddCommand(evm_state_exporter.Cmd)
+	rootCmd.AddCommand(verify_execution_result.Cmd)
 }
 
 func initConfig() {

cmd/util/cmd/verify_execution_result/cmd.go

@@ -0,0 +1,101 @@
+package verify
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/cobra"
+
+	"github.com/onflow/flow-go/engine/verification/verifier"
+	"github.com/onflow/flow-go/model/flow"
+)
+
+var (
+	flagLastK            uint64
+	flagDatadir          string
+	flagChunkDataPackDir string
+	flagChain            string
+	flagFromTo           string
+)
+
+// # verify the last 100 sealed blocks
+// ./util verify_execution_result --chain flow-testnet --datadir /var/flow/data/protocol --chunk_data_pack_dir /var/flow/data/chunk_data_pack --lastk 100
+// # verify the blocks from height 2000 to 3000
+// ./util verify_execution_result --chain flow-testnet --datadir /var/flow/data/protocol --chunk_data_pack_dir /var/flow/data/chunk_data_pack --from_to 2000-3000
+var Cmd = &cobra.Command{
+	Use:   "verify-execution-result",
+	Short: "verify block execution by verifying all chunks in the result",
+	Run:   run,
+}
+
+func init() {
+	Cmd.Flags().StringVar(&flagChain, "chain", "", "Chain name")
+	_ = Cmd.MarkFlagRequired("chain")
+
+	Cmd.Flags().StringVar(&flagDatadir, "datadir", "/var/flow/data/protocol",
+		"directory that stores the protocol state")
+	_ = Cmd.MarkFlagRequired("datadir")
+
+	Cmd.Flags().StringVar(&flagChunkDataPackDir, "chunk_data_pack_dir", "/var/flow/data/chunk_data_pack",
+		"directory that stores the protocol state")
+	_ = Cmd.MarkFlagRequired("chunk_data_pack_dir")
+
+	Cmd.Flags().Uint64Var(&flagLastK, "lastk", 1,
+		"last k sealed blocks to verify")
+
+	Cmd.Flags().StringVar(&flagFromTo, "from_to", "",
+		"the height range to verify blocks (inclusive), i.e, 1-1000, 1000-2000, 2000-3000, etc.")
+}
+
+func run(*cobra.Command, []string) {
+	chainID := flow.ChainID(flagChain)
+	_ = chainID.Chain()
+
+	if flagFromTo != "" {
+		from, to, err := parseFromTo(flagFromTo)
+		if err != nil {
+			log.Fatal().Err(err).Msg("could not parse from_to")
+		}
+
+		log.Info().Msgf("verifying range from %d to %d", from, to)
+		err = verifier.VerifyRange(from, to, chainID, flagDatadir, flagChunkDataPackDir)
+		if err != nil {
+			log.Fatal().Err(err).Msgf("could not verify range from %d to %d", from, to)
+		}
+		log.Info().Msgf("successfully verified range from %d to %d", from, to)
+
+	} else {
+		log.Info().Msgf("verifying last %d sealed blocks", flagLastK)
+		err := verifier.VerifyLastKHeight(flagLastK, chainID, flagDatadir, flagChunkDataPackDir)
+		if err != nil {
+			log.Fatal().Err(err).Msg("could not verify last k height")
+		}
+
+		log.Info().Msgf("successfully verified last %d sealed blocks", flagLastK)
+	}
+}
+
+func parseFromTo(fromTo string) (from, to uint64, err error) {
+	parts := strings.Split(fromTo, "-")
+	if len(parts) != 2 {
+		return 0, 0, fmt.Errorf("invalid format: expected 'from-to', got '%s'", fromTo)
+	}
+
+	from, err = strconv.ParseUint(strings.TrimSpace(parts[0]), 10, 64)
+	if err != nil {
+		return 0, 0, fmt.Errorf("invalid 'from' value: %w", err)
+	}
+
+	to, err = strconv.ParseUint(strings.TrimSpace(parts[1]), 10, 64)
+	if err != nil {
+		return 0, 0, fmt.Errorf("invalid 'to' value: %w", err)
+	}
+
+	if from > to {
+		return 0, 0, fmt.Errorf("'from' value (%d) must be less than or equal to 'to' value (%d)", from, to)
+	}
+
+	return from, to, nil
+}

engine/execution/computation/execution_verification_test.go

@@ -25,7 +25,6 @@ import (
 	"github.com/onflow/flow-go/engine/execution/testutil"
 	"github.com/onflow/flow-go/engine/execution/utils"
 	"github.com/onflow/flow-go/engine/testutil/mocklocal"
-	"github.com/onflow/flow-go/engine/verification/fetcher"
 	"github.com/onflow/flow-go/fvm"
 	"github.com/onflow/flow-go/fvm/blueprints"
 	"github.com/onflow/flow-go/fvm/environment"
@@ -36,6 +35,7 @@ import (
 	"github.com/onflow/flow-go/ledger/complete/wal/fixtures"
 	"github.com/onflow/flow-go/model/flow"
 	"github.com/onflow/flow-go/model/verification"
+	"github.com/onflow/flow-go/model/verification/convert"
 	"github.com/onflow/flow-go/module/chunks"
 	"github.com/onflow/flow-go/module/executiondatasync/execution_data"
 	exedataprovider "github.com/onflow/flow-go/module/executiondatasync/provider"
@@ -69,7 +69,7 @@ func Test_ExecutionMatchesVerification(t *testing.T) {
 			`access(all) contract Foo {
 				access(all) event FooEvent(x: Int, y: Int)
 
-				access(all) fun emitEvent() { 
+				access(all) fun emitEvent() {
 					emit FooEvent(x: 2, y: 1)
 				}
 			}`), "Foo")
@@ -113,7 +113,7 @@ func Test_ExecutionMatchesVerification(t *testing.T) {
 			`access(all) contract Foo {
 				access(all) event FooEvent(x: Int, y: Int)
 
-				access(all) fun emitEvent() { 
+				access(all) fun emitEvent() {
 					emit FooEvent(x: 2, y: 1)
 				}
 			}`), "Foo")
@@ -585,34 +585,34 @@ func TestTransactionFeeDeduction(t *testing.T) {
 							//
 							// The withdraw amount and the account from getAccount
 							// would be the parameters to the transaction
-							
+
 							import FungibleToken from 0x%s
 							import FlowToken from 0x%s
-							
+
 							transaction(amount: UFix64, to: Address) {
-							
+
 								// The Vault resource that holds the tokens that are being transferred
 								let sentVault: @{FungibleToken.Vault}
-							
+
 								prepare(signer: auth(BorrowValue) &Account) {
-							
+
 									// Get a reference to the signer's stored vault
 									let vaultRef = signer.storage.borrow<auth(FungibleToken.Withdraw) &FlowToken.Vault>(from: /storage/flowTokenVault)
 										?? panic("Could not borrow reference to the owner's Vault!")
-							
+
 									// Withdraw tokens from the signer's stored vault
 									self.sentVault <- vaultRef.withdraw(amount: amount)
 								}
-							
+
 								execute {
-							
+
 									// Get the recipient's public account object
 									let recipient = getAccount(to)
-							
+
 									// Get a reference to the recipient's Receiver
 									let receiverRef = recipient.capabilities.borrow<&{FungibleToken.Receiver}>(/public/flowTokenReceiver)
 										?? panic("Could not borrow receiver reference to the recipient's Vault")
-							
+
 									// Deposit the withdrawn tokens in the recipient's receiver
 									receiverRef.deposit(from: <-self.sentVault)
 								}
@@ -840,7 +840,7 @@ func executeBlockAndVerifyWithParameters(t *testing.T,
 
 	for i, chunk := range er.Chunks {
 		isSystemChunk := i == er.Chunks.Len()-1
-		offsetForChunk, err := fetcher.TransactionOffsetForChunk(er.Chunks, chunk.Index)
+		offsetForChunk, err := convert.TransactionOffsetForChunk(er.Chunks, chunk.Index)
 		require.NoError(t, err)
 
 		vcds[i] = &verification.VerifiableChunkData{

engine/verification/fetcher/engine.go

@@ -12,6 +12,7 @@ import (
 	"github.com/onflow/flow-go/model/flow"
 	"github.com/onflow/flow-go/model/flow/filter"
 	"github.com/onflow/flow-go/model/verification"
+	"github.com/onflow/flow-go/model/verification/convert"
 	"github.com/onflow/flow-go/module"
 	"github.com/onflow/flow-go/module/mempool"
 	"github.com/onflow/flow-go/module/trace"
@@ -259,7 +260,7 @@ func (e *Engine) HandleChunkDataPack(originID flow.Identifier, response *verific
 		Uint64("block_height", status.BlockHeight).
 		Hex("result_id", logging.ID(resultID)).
 		Uint64("chunk_index", status.ChunkIndex).
-		Bool("system_chunk", IsSystemChunk(status.ChunkIndex, status.ExecutionResult)).
+		Bool("system_chunk", convert.IsSystemChunk(status.ChunkIndex, status.ExecutionResult)).
 		Logger()
 
 	span, ctx := e.tracer.StartBlockSpan(context.Background(), status.ExecutionResult.BlockID, trace.VERFetcherHandleChunkDataPack)
@@ -413,7 +414,7 @@ func (e Engine) validateCollectionID(
 	result *flow.ExecutionResult,
 	chunk *flow.Chunk) error {
 
-	if IsSystemChunk(chunk.Index, result) {
+	if convert.IsSystemChunk(chunk.Index, result) {
 		return e.validateSystemChunkCollection(chunkDataPack)
 	}
 
@@ -550,29 +551,13 @@ func (e *Engine) makeVerifiableChunkData(chunk *flow.Chunk,
 	chunkDataPack *flow.ChunkDataPack,
 ) (*verification.VerifiableChunkData, error) {
 
-	// system chunk is the last chunk
-	isSystemChunk := IsSystemChunk(chunk.Index, result)
-
-	endState, err := EndStateCommitment(result, chunk.Index, isSystemChunk)
-	if err != nil {
-		return nil, fmt.Errorf("could not compute end state of chunk: %w", err)
-	}
-
-	transactionOffset, err := TransactionOffsetForChunk(result.Chunks, chunk.Index)
-	if err != nil {
-		return nil, fmt.Errorf("cannot compute transaction offset for chunk: %w", err)
-	}
-
-	return &verification.VerifiableChunkData{
-		IsSystemChunk:     isSystemChunk,
-		Chunk:             chunk,
-		Header:            header,
-		Snapshot:          snapshot,
-		Result:            result,
-		ChunkDataPack:     chunkDataPack,
-		EndState:          endState,
-		TransactionOffset: transactionOffset,
-	}, nil
+	return convert.FromChunkDataPack(
+		chunk,
+		chunkDataPack,
+		header,
+		snapshot,
+		result,
+	)
 }
 
 // requestChunkDataPack creates and dispatches a chunk data pack request to the requester engine.
@@ -661,42 +646,3 @@ func executorsOf(receipts []*flow.ExecutionReceipt, resultID flow.Identifier) (f
 
 	return agrees, disagrees
 }
-
-// EndStateCommitment computes the end state of the given chunk.
-func EndStateCommitment(result *flow.ExecutionResult, chunkIndex uint64, systemChunk bool) (flow.StateCommitment, error) {
-	var endState flow.StateCommitment
-	if systemChunk {
-		var err error
-		// last chunk in a result is the system chunk and takes final state commitment
-		endState, err = result.FinalStateCommitment()
-		if err != nil {
-			return flow.DummyStateCommitment, fmt.Errorf("can not read final state commitment, likely a bug:%w", err)
-		}
-	} else {
-		// any chunk except last takes the subsequent chunk's start state
-		endState = result.Chunks[chunkIndex+1].StartState
-	}
-
-	return endState, nil
-}
-
-// TransactionOffsetForChunk calculates transaction offset for a given chunk which is the index of the first
-// transaction of this chunk within the whole block
-func TransactionOffsetForChunk(chunks flow.ChunkList, chunkIndex uint64) (uint32, error) {
-	if int(chunkIndex) > len(chunks)-1 {
-		return 0, fmt.Errorf("chunk list out of bounds, len %d asked for chunk %d", len(chunks), chunkIndex)
-	}
-	var offset uint32 = 0
-	for i := 0; i < int(chunkIndex); i++ {
-		offset += uint32(chunks[i].NumberOfTransactions)
-	}
-	return offset, nil
-}
-
-// IsSystemChunk returns true if `chunkIndex` points to a system chunk in `result`.
-// Otherwise, it returns false.
-// In the current version, a chunk is a system chunk if it is the last chunk of the
-// execution result.
-func IsSystemChunk(chunkIndex uint64, result *flow.ExecutionResult) bool {
-	return chunkIndex == uint64(len(result.Chunks)-1)
-}