Merge branch 'master' into leo/refactor-index-result

Author: zhangchiqing

.github/workflows/ci.yml

@@ -62,8 +62,6 @@ jobs:
         uses: golangci/golangci-lint-action@v6
         continue-on-error: true # after installation (what we care about), this step will fail - this line allows workflow to continue
         with:
-          # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-          version: v1.63
           args: "--no-config --disable-all" # set args so that no linters are actually run
       - name: Build custom linter binary
         if: steps.cache-linter.outputs.cache-hit != 'true'

cmd/access/node_builder/access_node_builder.go

@@ -66,7 +66,6 @@ import (
 	"github.com/onflow/flow-go/engine/common/version"
 	"github.com/onflow/flow-go/engine/execution/computation"
 	"github.com/onflow/flow-go/engine/execution/computation/query"
-	"github.com/onflow/flow-go/fvm"
 	"github.com/onflow/flow-go/fvm/storage/derived"
 	"github.com/onflow/flow-go/ledger"
 	"github.com/onflow/flow-go/ledger/complete/wal"
@@ -183,7 +182,6 @@ type AccessNodeConfig struct {
 	storeTxResultErrorMessages           bool
 	stopControlEnabled                   bool
 	registerDBPruneThreshold             uint64
-	scheduledCallbacksEnabled            bool
 }
 
 type PublicNetworkConfig struct {
@@ -291,7 +289,6 @@ func DefaultAccessNodeConfig() *AccessNodeConfig {
 		storeTxResultErrorMessages:           false,
 		stopControlEnabled:                   false,
 		registerDBPruneThreshold:             0,
-		scheduledCallbacksEnabled:            fvm.DefaultScheduledCallbacksEnabled,
 	}
 }
 
@@ -1324,10 +1321,6 @@ func (builder *FlowAccessNodeBuilder) extraFlags() {
 			"stop-control-enabled",
 			defaultConfig.stopControlEnabled,
 			"whether to enable the stop control feature. Default value is false")
-		flags.BoolVar(&builder.scheduledCallbacksEnabled,
-			"scheduled-callbacks-enabled",
-			defaultConfig.scheduledCallbacksEnabled,
-			"whether to include scheduled callback transactions in system collections.")
 		// ExecutionDataRequester config
 		flags.BoolVar(&builder.executionDataSyncEnabled,
 			"execution-data-sync-enabled",
@@ -1517,6 +1510,10 @@ func (builder *FlowAccessNodeBuilder) extraFlags() {
 		flags.BoolVar(&unusedRetryEnabled, "retry-enabled", false, "[deprecated] whether to enable the retry mechanism at the access node level")
 		_ = flags.MarkDeprecated("retry-enabled", "[deprecated] this flag is ignored and will be removed in a future release.")
 
+		var unusedScheduledCallbacksEnabled bool
+		flags.BoolVar(&unusedScheduledCallbacksEnabled, "scheduled-callbacks-enabled", false, "[deprecated] whether to include scheduled callback transactions in system collections.")
+		_ = flags.MarkDeprecated("scheduled-callbacks-enabled", "[deprecated] this flag is ignored and will be removed in a future release.")
+
 	}).ValidateFlags(func() error {
 		if builder.supportsObserver && (builder.PublicNetworkConfig.BindAddress == cmd.NotSet || builder.PublicNetworkConfig.BindAddress == "") {
 			return errors.New("public-network-address must be set if supports-observer is true")
@@ -2160,7 +2157,6 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) {
 				ExecNodeIdentitiesProvider: notNil(builder.ExecNodeIdentitiesProvider),
 				TxErrorMessageProvider:     notNil(builder.txResultErrorMessageProvider),
 				MaxScriptAndArgumentSize:   config.BackendConfig.AccessConfig.MaxRequestMsgSize,
-				ScheduledCallbacksEnabled:  builder.scheduledCallbacksEnabled,
 			})
 			if err != nil {
 				return nil, fmt.Errorf("could not initialize backend: %w", err)

cmd/execution_config.go

@@ -145,7 +145,7 @@ func (exeConf *ExecutionConfig) SetupFlags(flags *pflag.FlagSet) {
 	flags.BoolVar(&exeConf.onflowOnlyLNs, "temp-onflow-only-lns", false, "do not use unless required. forces node to only request collections from onflow collection nodes")
 	flags.BoolVar(&exeConf.enableStorehouse, "enable-storehouse", false, "enable storehouse to store registers on disk, default is false")
 	flags.BoolVar(&exeConf.enableChecker, "enable-checker", true, "enable checker to check the correctness of the execution result, default is true")
-	flags.BoolVar(&exeConf.scheduleCallbacksEnabled, "scheduled-callbacks-enabled", fvm.DefaultScheduledCallbacksEnabled, "enable execution of scheduled callbacks")
+	flags.BoolVar(&exeConf.scheduleCallbacksEnabled, "scheduled-callbacks-enabled", fvm.DefaultScheduledTransactionsEnabled, "[deprecated] enable execution of scheduled transactions")
 	// deprecated. Retain it to prevent nodes that previously had this configuration from crashing.
 	var deprecatedEnableNewIngestionEngine bool
 	flags.BoolVar(&deprecatedEnableNewIngestionEngine, "enable-new-ingestion-engine", true, "enable new ingestion engine, default is true")

cmd/util/cmd/verify_execution_result/cmd.go

@@ -16,15 +16,15 @@ import (
 )
 
 var (
-	flagLastK                     uint64
-	flagDatadir                   string
-	flagChunkDataPackDir          string
-	flagChain                     string
-	flagFromTo                    string
-	flagWorkerCount               uint // number of workers to verify the blocks concurrently
-	flagStopOnMismatch            bool
-	flagtransactionFeesDisabled   bool
-	flagScheduledCallbacksEnabled bool
+	flagLastK                        uint64
+	flagDatadir                      string
+	flagChunkDataPackDir             string
+	flagChain                        string
+	flagFromTo                       string
+	flagWorkerCount                  uint // number of workers to verify the blocks concurrently
+	flagStopOnMismatch               bool
+	flagtransactionFeesDisabled      bool
+	flagScheduledTransactionsEnabled bool
 )
 
 // # verify the last 100 sealed blocks
@@ -60,7 +60,7 @@ func init() {
 
 	Cmd.Flags().BoolVar(&flagtransactionFeesDisabled, "fees_disabled", false, "disable transaction fees")
 
-	Cmd.Flags().BoolVar(&flagScheduledCallbacksEnabled, "scheduled_callbacks_enabled", fvm.DefaultScheduledCallbacksEnabled, "enable scheduled callbacks")
+	Cmd.Flags().BoolVar(&flagScheduledTransactionsEnabled, "scheduled_callbacks_enabled", fvm.DefaultScheduledTransactionsEnabled, "[deprecated] enable scheduled transactions")
 }
 
 func run(*cobra.Command, []string) {
@@ -95,14 +95,14 @@ func run(*cobra.Command, []string) {
 		}
 
 		lg.Info().Msgf("verifying range from %d to %d", from, to)
-		err = verifier.VerifyRange(lockManager, from, to, chainID, flagDatadir, flagChunkDataPackDir, flagWorkerCount, flagStopOnMismatch, flagtransactionFeesDisabled, flagScheduledCallbacksEnabled)
+		err = verifier.VerifyRange(lockManager, from, to, chainID, flagDatadir, flagChunkDataPackDir, flagWorkerCount, flagStopOnMismatch, flagtransactionFeesDisabled, flagScheduledTransactionsEnabled)
 		if err != nil {
 			lg.Fatal().Err(err).Msgf("could not verify range from %d to %d", from, to)
 		}
 		lg.Info().Msgf("finished verified range from %d to %d", from, to)
 	} else {
 		lg.Info().Msgf("verifying last %d sealed blocks", flagLastK)
-		err := verifier.VerifyLastKHeight(lockManager, flagLastK, chainID, flagDatadir, flagChunkDataPackDir, flagWorkerCount, flagStopOnMismatch, flagtransactionFeesDisabled, flagScheduledCallbacksEnabled)
+		err := verifier.VerifyLastKHeight(lockManager, flagLastK, chainID, flagDatadir, flagChunkDataPackDir, flagWorkerCount, flagStopOnMismatch, flagtransactionFeesDisabled, flagScheduledTransactionsEnabled)
 		if err != nil {
 			lg.Fatal().Err(err).Msg("could not verify last k height")
 		}

cmd/util/ledger/reporters/fungible_token_tracker_test.go

@@ -4,7 +4,6 @@ import (
 	"encoding/hex"
 	"fmt"
 	"os"
-	"strings"
 	"testing"
 
 	"github.com/onflow/cadence"
@@ -170,13 +169,13 @@ func TestFungibleTokenTracker(t *testing.T) {
 	require.NoError(t, err)
 
 	// wrappedToken
-	require.True(t, strings.Contains(string(data), `{"path":"storage/wrappedToken/vault","address":"8c5303eaa26202d6","balance":105,"type_id":"A.7e60df042a9c0868.FlowToken.Vault"}`))
+	require.Contains(t, string(data), `{"path":"storage/wrappedToken/vault","address":"8c5303eaa26202d6","balance":105,"type_id":"A.7e60df042a9c0868.FlowToken.Vault"}`)
 	// flowTokenVaults
-	require.True(t, strings.Contains(string(data), `{"path":"storage/flowTokenVault","address":"8c5303eaa26202d6","balance":99999999999599895,"type_id":"A.7e60df042a9c0868.FlowToken.Vault"}`))
-	require.True(t, strings.Contains(string(data), `{"path":"storage/flowTokenVault","address":"9a0766d93b6608b7","balance":100000,"type_id":"A.7e60df042a9c0868.FlowToken.Vault"}`))
-	require.True(t, strings.Contains(string(data), `{"path":"storage/flowTokenVault","address":"7e60df042a9c0868","balance":100000,"type_id":"A.7e60df042a9c0868.FlowToken.Vault"}`))
-	require.True(t, strings.Contains(string(data), `{"path":"storage/flowTokenVault","address":"912d5440f7e3769e","balance":100000,"type_id":"A.7e60df042a9c0868.FlowToken.Vault"}`))
-	require.True(t, strings.Contains(string(data), `{"path":"storage/flowTokenVault","address":"754aed9de6197641","balance":100000,"type_id":"A.7e60df042a9c0868.FlowToken.Vault"}`))
+	require.Contains(t, string(data), `{"path":"storage/flowTokenVault","address":"8c5303eaa26202d6","balance":99999999999599895,"type_id":"A.7e60df042a9c0868.FlowToken.Vault"}`)
+	require.Contains(t, string(data), `{"path":"storage/flowTokenVault","address":"9a0766d93b6608b7","balance":100000,"type_id":"A.7e60df042a9c0868.FlowToken.Vault"}`)
+	require.Contains(t, string(data), `{"path":"storage/flowTokenVault","address":"7e60df042a9c0868","balance":100000,"type_id":"A.7e60df042a9c0868.FlowToken.Vault"}`)
+	require.Contains(t, string(data), `{"path":"storage/flowTokenVault","address":"912d5440f7e3769e","balance":100000,"type_id":"A.7e60df042a9c0868.FlowToken.Vault"}`)
+	require.Contains(t, string(data), `{"path":"storage/flowTokenVault","address":"754aed9de6197641","balance":100000,"type_id":"A.7e60df042a9c0868.FlowToken.Vault"}`)
 
 	// do not remove this line, see https://github.com/onflow/flow-go/pull/2237
 	t.Log("success")

cmd/verification_builder.go

@@ -54,8 +54,8 @@ type VerificationConfig struct {
 	blockWorkers uint64 // number of blocks processed in parallel.
 	chunkWorkers uint64 // number of chunks processed in parallel.
 
-	stopAtHeight              uint64 // height to stop the node on
-	scheduledCallbacksEnabled bool   // enable execution of scheduled callbacks
+	stopAtHeight                 uint64 // height to stop the node on
+	scheduledTransactionsEnabled bool   // enable execution of scheduled transactions
 }
 
 type VerificationNodeBuilder struct {
@@ -83,7 +83,7 @@ func (v *VerificationNodeBuilder) LoadFlags() {
 			flags.Uint64Var(&v.verConf.blockWorkers, "block-workers", blockconsumer.DefaultBlockWorkers, "maximum number of blocks being processed in parallel")
 			flags.Uint64Var(&v.verConf.chunkWorkers, "chunk-workers", chunkconsumer.DefaultChunkWorkers, "maximum number of execution nodes a chunk data pack request is dispatched to")
 			flags.Uint64Var(&v.verConf.stopAtHeight, "stop-at-height", 0, "height to stop the node at (0 to disable)")
-			flags.BoolVar(&v.verConf.scheduledCallbacksEnabled, "scheduled-callbacks-enabled", fvm.DefaultScheduledCallbacksEnabled, "enable execution of scheduled callbacks")
+			flags.BoolVar(&v.verConf.scheduledTransactionsEnabled, "scheduled-callbacks-enabled", fvm.DefaultScheduledTransactionsEnabled, "enable execution of scheduled transactions")
 		})
 }
 
@@ -211,7 +211,7 @@ func (v *VerificationNodeBuilder) LoadComponentsAndModules() {
 				computation.DefaultFVMOptions(
 					node.RootChainID,
 					false,
-					v.verConf.scheduledCallbacksEnabled,
+					v.verConf.scheduledTransactionsEnabled,
 				)...,
 			)
 			vmCtx := fvm.NewContext(fvmOptions...)

engine/access/rpc/backend/backend.go

@@ -25,13 +25,13 @@ import (
 	"github.com/onflow/flow-go/engine/access/rpc/backend/transactions/provider"
 	"github.com/onflow/flow-go/engine/access/rpc/backend/transactions/status"
 	txstream "github.com/onflow/flow-go/engine/access/rpc/backend/transactions/stream"
-	"github.com/onflow/flow-go/engine/access/rpc/backend/transactions/system"
 	"github.com/onflow/flow-go/engine/access/rpc/connection"
 	"github.com/onflow/flow-go/engine/access/subscription"
 	"github.com/onflow/flow-go/engine/access/subscription/tracker"
 	"github.com/onflow/flow-go/engine/common/rpc"
 	"github.com/onflow/flow-go/engine/common/version"
 	accessmodel "github.com/onflow/flow-go/model/access"
+	"github.com/onflow/flow-go/model/access/systemcollection"
 	"github.com/onflow/flow-go/model/flow"
 	"github.com/onflow/flow-go/module"
 	"github.com/onflow/flow-go/module/counters"
@@ -118,7 +118,6 @@ type Params struct {
 	VersionControl             *version.VersionControl
 	ExecNodeIdentitiesProvider *rpc.ExecutionNodeIdentitiesProvider
 	TxErrorMessageProvider     error_messages.Provider
-	ScheduledCallbacksEnabled  bool
 }
 
 var _ access.API = (*Backend)(nil)
@@ -138,9 +137,9 @@ func New(params Params) (*Backend, error) {
 		}
 	}
 
-	systemCollection, err := system.DefaultSystemCollection(params.ChainID, params.ScheduledCallbacksEnabled)
+	systemCollections, err := systemcollection.NewVersioned(params.ChainID.Chain(), systemcollection.Default(params.ChainID))
 	if err != nil {
-		return nil, fmt.Errorf("failed to construct system collection: %w", err)
+		return nil, fmt.Errorf("failed to create system collection set: %w", err)
 	}
 
 	accountsBackend, err := accounts.NewAccountsBackend(
@@ -220,10 +219,9 @@ func New(params Params) (*Backend, error) {
 		params.EventsIndex,
 		params.TxResultsIndex,
 		params.TxErrorMessageProvider,
-		systemCollection,
+		systemCollections,
 		txStatusDeriver,
 		params.ChainID,
-		params.ScheduledCallbacksEnabled,
 	)
 	execNodeTxProvider := provider.NewENTransactionProvider(
 		params.Log,
@@ -233,34 +231,32 @@ func New(params Params) (*Backend, error) {
 		params.Communicator,
 		params.ExecNodeIdentitiesProvider,
 		txStatusDeriver,
-		systemCollection,
+		systemCollections,
 		params.ChainID,
-		params.ScheduledCallbacksEnabled,
 	)
 	failoverTxProvider := provider.NewFailoverTransactionProvider(localTxProvider, execNodeTxProvider)
 
 	txParams := transactions.Params{
-		Log:                          params.Log,
-		Metrics:                      params.AccessMetrics,
-		State:                        params.State,
-		ChainID:                      params.ChainID,
-		SystemCollection:             systemCollection,
-		StaticCollectionRPCClient:    params.CollectionRPC,
-		HistoricalAccessNodeClients:  params.HistoricalAccessNodes,
-		NodeCommunicator:             params.Communicator,
-		ConnFactory:                  params.ConnFactory,
-		NodeProvider:                 params.ExecNodeIdentitiesProvider,
-		Blocks:                       params.Blocks,
-		Collections:                  params.Collections,
-		Transactions:                 params.Transactions,
-		TxErrorMessageProvider:       params.TxErrorMessageProvider,
-		ScheduledTransactions:        params.ScheduledTransactions,
-		TxResultCache:                txResCache,
-		TxValidator:                  txValidator,
-		TxStatusDeriver:              txStatusDeriver,
-		EventsIndex:                  params.EventsIndex,
-		TxResultsIndex:               params.TxResultsIndex,
-		ScheduledTransactionsEnabled: params.ScheduledCallbacksEnabled,
+		Log:                         params.Log,
+		Metrics:                     params.AccessMetrics,
+		State:                       params.State,
+		ChainID:                     params.ChainID,
+		SystemCollections:           systemCollections,
+		StaticCollectionRPCClient:   params.CollectionRPC,
+		HistoricalAccessNodeClients: params.HistoricalAccessNodes,
+		NodeCommunicator:            params.Communicator,
+		ConnFactory:                 params.ConnFactory,
+		NodeProvider:                params.ExecNodeIdentitiesProvider,
+		Blocks:                      params.Blocks,
+		Collections:                 params.Collections,
+		Transactions:                params.Transactions,
+		TxErrorMessageProvider:      params.TxErrorMessageProvider,
+		ScheduledTransactions:       params.ScheduledTransactions,
+		TxResultCache:               txResCache,
+		TxValidator:                 txValidator,
+		TxStatusDeriver:             txStatusDeriver,
+		EventsIndex:                 params.EventsIndex,
+		TxResultsIndex:              params.TxResultsIndex,
 	}
 
 	switch params.TxResultQueryMode {

engine/access/rpc/backend/backend_test.go

@@ -922,7 +922,7 @@ func (suite *Suite) TestGetTransactionResultByIndex() {
 	suite.state.On("Sealed").Return(suite.snapshot, nil).Maybe()
 
 	ctx := context.Background()
-	block := unittest.BlockFixture()
+	block := unittest.BlockFixture(unittest.Block.WithHeight(1))
 	blockId := block.ID()
 	index := uint32(0)
 
@@ -957,6 +957,12 @@ func (suite *Suite) TestGetTransactionResultByIndex() {
 		On("GetTransactionResultByIndex", mock.Anything, exeEventReq).
 		Return(exeEventResp, nil)
 
+	suite.execClient.
+		On("GetEventsForBlockIDs", mock.Anything, mock.Anything).
+		Return(&execproto.GetEventsForBlockIDsResponse{
+			Results: []*execproto.GetEventsForBlockIDsResponse_Result{},
+		}, nil)
+
 	suite.Run("TestGetTransactionResultByIndex - happy path", func() {
 		suite.snapshot.On("Head").Return(block.ToHeader(), nil).Once()
 		result, err := backend.GetTransactionResultByIndex(ctx, blockId, index, entitiesproto.EventEncodingVersion_JSON_CDC_V0)
@@ -1008,7 +1014,7 @@ func (suite *Suite) TestGetTransactionResultsByBlockID() {
 	}
 
 	exeEventResp := &execproto.GetTransactionResultsResponse{
-		TransactionResults:   []*execproto.GetTransactionResultResponse{{}},
+		TransactionResults:   []*execproto.GetTransactionResultResponse{{}, {}},
 		EventEncodingVersion: entitiesproto.EventEncodingVersion_CCF_V0,
 	}
 
@@ -1017,11 +1023,17 @@ func (suite *Suite) TestGetTransactionResultsByBlockID() {
 	params := suite.defaultBackendParams()
 	// the connection factory should be used to get the execution node client
 	params.ConnFactory = suite.setupConnectionFactory()
-	params.ScheduledCallbacksEnabled = true
 
 	backend, err := New(params)
 	suite.Require().NoError(err)
 
+	suite.execClient.
+		On("GetEventsForBlockIDs", mock.Anything, mock.Anything).
+		Return(&execproto.GetEventsForBlockIDsResponse{
+			Results: []*execproto.GetEventsForBlockIDsResponse_Result{},
+		}, nil).
+		Maybe()
+
 	suite.execClient.
 		On("GetTransactionResultsByBlockID", mock.Anything, exeEventReq).
 		Return(exeEventResp, nil)
@@ -1848,7 +1860,7 @@ func (suite *Suite) TestGetTransactionResultByIndexAndBlockIdEventEncodingVersio
 	suite.state.On("Sealed").Return(suite.snapshot, nil).Maybe()
 
 	ctx := context.Background()
-	block := unittest.BlockFixture()
+	block := unittest.BlockFixture(unittest.Block.WithHeight(1))
 	blockId := block.ID()
 	index := uint32(0)
 
@@ -1872,6 +1884,14 @@ func (suite *Suite) TestGetTransactionResultByIndexAndBlockIdEventEncodingVersio
 	backend, err := New(params)
 	suite.Require().NoError(err)
 
+	// Mock GetEventsForBlockIDs for system collection construction
+	suite.execClient.
+		On("GetEventsForBlockIDs", mock.Anything, mock.Anything).
+		Return(&execproto.GetEventsForBlockIDsResponse{
+			Results: []*execproto.GetEventsForBlockIDsResponse_Result{},
+		}, nil).
+		Maybe()
+
 	exeNodeEventEncodingVersion := entitiesproto.EventEncodingVersion_CCF_V0
 	ccfEvents, jsoncdcEvents := generateEncodedEvents(suite.T(), 1)
 	eventMessages := convert.EventsToMessages(ccfEvents)
@@ -1909,6 +1929,10 @@ func (suite *Suite) TestGetTransactionResultByIndexAndBlockIdEventEncodingVersio
 		suite.Run(fmt.Sprintf("test %s event encoding version for GetTransactionResultsByBlockID", version.String()), func() {
 			exeEventResp := &execproto.GetTransactionResultsResponse{
 				TransactionResults: []*execproto.GetTransactionResultResponse{
+					{
+						Events:               eventMessages,
+						EventEncodingVersion: exeNodeEventEncodingVersion,
+					},
 					{
 						Events:               eventMessages,
 						EventEncodingVersion: exeNodeEventEncodingVersion,