Merge branch 'master' into jordan/existence-check-snapshot

Author: jordanschalm

engine/access/access_test.go

@@ -39,7 +39,8 @@ import (
 	"github.com/onflow/flow-go/network/channels"
 	"github.com/onflow/flow-go/network/mocknetwork"
 	protocol "github.com/onflow/flow-go/state/protocol/mock"
-	storage "github.com/onflow/flow-go/storage/badger"
+	"github.com/onflow/flow-go/storage"
+	bstorage "github.com/onflow/flow-go/storage/badger"
 	"github.com/onflow/flow-go/storage/badger/operation"
 	"github.com/onflow/flow-go/storage/util"
 	"github.com/onflow/flow-go/utils/unittest"
@@ -109,23 +110,20 @@ func (suite *Suite) SetupTest() {
 }
 
 func (suite *Suite) RunTest(
-	f func(handler *access.Handler, db *badger.DB, blocks *storage.Blocks, headers *storage.Headers, results *storage.ExecutionResults),
+	f func(handler *access.Handler, db *badger.DB, all *storage.All),
 ) {
 	unittest.RunWithBadgerDB(suite.T(), func(db *badger.DB) {
-		headers, _, _, _, _, blocks, _, _, _, _, results := util.StorageLayer(suite.T(), db)
-		transactions := storage.NewTransactions(suite.metrics, db)
-		collections := storage.NewCollections(db, transactions)
-		receipts := storage.NewExecutionReceipts(suite.metrics, db, results, storage.DefaultCacheSize)
+		all := util.StorageLayer(suite.T(), db)
 
 		suite.backend = backend.New(suite.state,
 			suite.collClient,
 			nil,
-			blocks,
-			headers,
-			collections,
-			transactions,
-			receipts,
-			results,
+			all.Blocks,
+			all.Headers,
+			all.Collections,
+			all.Transactions,
+			all.Receipts,
+			all.Results,
 			suite.chainID,
 			suite.metrics,
 			nil,
@@ -138,12 +136,12 @@ func (suite *Suite) RunTest(
 		)
 
 		handler := access.NewHandler(suite.backend, suite.chainID.Chain(), access.WithBlockSignerDecoder(suite.signerIndicesDecoder))
-		f(handler, db, blocks, headers, results)
+		f(handler, db, all)
 	})
 }
 
 func (suite *Suite) TestSendAndGetTransaction() {
-	suite.RunTest(func(handler *access.Handler, _ *badger.DB, _ *storage.Blocks, _ *storage.Headers, _ *storage.ExecutionResults) {
+	suite.RunTest(func(handler *access.Handler, _ *badger.DB, _ *storage.All) {
 		referenceBlock := unittest.BlockHeaderFixture()
 		transaction := unittest.TransactionFixture()
 		transaction.SetReferenceBlockID(referenceBlock.ID())
@@ -196,7 +194,7 @@ func (suite *Suite) TestSendAndGetTransaction() {
 }
 
 func (suite *Suite) TestSendExpiredTransaction() {
-	suite.RunTest(func(handler *access.Handler, _ *badger.DB, _ *storage.Blocks, _ *storage.Headers, _ *storage.ExecutionResults) {
+	suite.RunTest(func(handler *access.Handler, _ *badger.DB, _ *storage.All) {
 		referenceBlock := unittest.BlockHeaderFixture()
 
 		// create latest block that is past the expiry window
@@ -251,8 +249,8 @@ func (suite *Suite) TestSendTransactionToRandomCollectionNode() {
 
 		// create storage
 		metrics := metrics.NewNoopCollector()
-		transactions := storage.NewTransactions(metrics, db)
-		collections := storage.NewCollections(db, transactions)
+		transactions := bstorage.NewTransactions(metrics, db)
+		collections := bstorage.NewCollections(db, transactions)
 
 		// create collection node cluster
 		count := 2
@@ -349,16 +347,16 @@ func (suite *Suite) TestSendTransactionToRandomCollectionNode() {
 }
 
 func (suite *Suite) TestGetBlockByIDAndHeight() {
-	suite.RunTest(func(handler *access.Handler, db *badger.DB, blocks *storage.Blocks, _ *storage.Headers, _ *storage.ExecutionResults) {
+	suite.RunTest(func(handler *access.Handler, db *badger.DB, all *storage.All) {
 
 		// test block1 get by ID
 		block1 := unittest.BlockFixture()
 		// test block2 get by height
 		block2 := unittest.BlockFixture()
 		block2.Header.Height = 2
 
-		require.NoError(suite.T(), blocks.Store(&block1))
-		require.NoError(suite.T(), blocks.Store(&block2))
+		require.NoError(suite.T(), all.Blocks.Store(&block1))
+		require.NoError(suite.T(), all.Blocks.Store(&block2))
 
 		// the follower logic should update height index on the block storage when a block is finalized
 		err := db.Update(operation.IndexBlockHeight(block2.Header.Height, block2.ID()))
@@ -473,7 +471,7 @@ func (suite *Suite) TestGetBlockByIDAndHeight() {
 }
 
 func (suite *Suite) TestGetExecutionResultByBlockID() {
-	suite.RunTest(func(handler *access.Handler, db *badger.DB, blocks *storage.Blocks, _ *storage.Headers, executionResults *storage.ExecutionResults) {
+	suite.RunTest(func(handler *access.Handler, db *badger.DB, all *storage.All) {
 
 		// test block1 get by ID
 		nonexistingID := unittest.IdentifierFixture()
@@ -483,8 +481,8 @@ func (suite *Suite) TestGetExecutionResultByBlockID() {
 			unittest.WithExecutionResultBlockID(blockID),
 			unittest.WithServiceEvents(2))
 
-		require.NoError(suite.T(), executionResults.Store(er))
-		require.NoError(suite.T(), executionResults.Index(blockID, er.ID()))
+		require.NoError(suite.T(), all.Results.Store(er))
+		require.NoError(suite.T(), all.Results.Index(blockID, er.ID()))
 
 		assertResp := func(resp *accessproto.ExecutionResultForBlockIDResponse, err error, executionResult *flow.ExecutionResult) {
 			require.NoError(suite.T(), err)
@@ -555,9 +553,9 @@ func (suite *Suite) TestGetExecutionResultByBlockID() {
 // is reported as sealed
 func (suite *Suite) TestGetSealedTransaction() {
 	unittest.RunWithBadgerDB(suite.T(), func(db *badger.DB) {
-		headers, _, _, _, _, blocks, _, _, _, _, _ := util.StorageLayer(suite.T(), db)
-		results := storage.NewExecutionResults(suite.metrics, db)
-		receipts := storage.NewExecutionReceipts(suite.metrics, db, results, storage.DefaultCacheSize)
+		all := util.StorageLayer(suite.T(), db)
+		results := bstorage.NewExecutionResults(suite.metrics, db)
+		receipts := bstorage.NewExecutionReceipts(suite.metrics, db, results, bstorage.DefaultCacheSize)
 		enIdentities := unittest.IdentityListFixture(2, unittest.WithRole(flow.RoleExecution))
 		enNodeIDs := flow.IdentifierList(enIdentities.NodeIDs())
 
@@ -594,8 +592,8 @@ func (suite *Suite) TestGetSealedTransaction() {
 
 		// initialize storage
 		metrics := metrics.NewNoopCollector()
-		transactions := storage.NewTransactions(metrics, db)
-		collections := storage.NewCollections(db, transactions)
+		transactions := bstorage.NewTransactions(metrics, db)
+		collections := bstorage.NewCollections(db, transactions)
 		collectionsToMarkFinalized, err := stdmap.NewTimes(100)
 		require.NoError(suite.T(), err)
 		collectionsToMarkExecuted, err := stdmap.NewTimes(100)
@@ -606,8 +604,8 @@ func (suite *Suite) TestGetSealedTransaction() {
 		backend := backend.New(suite.state,
 			suite.collClient,
 			nil,
-			blocks,
-			headers,
+			all.Blocks,
+			all.Headers,
 			collections,
 			transactions,
 			receipts,
@@ -625,19 +623,19 @@ func (suite *Suite) TestGetSealedTransaction() {
 
 		handler := access.NewHandler(backend, suite.chainID.Chain())
 
-		rpcEngBuilder, err := rpc.NewBuilder(suite.log, suite.state, rpc.Config{}, nil, nil, blocks, headers, collections, transactions, receipts,
+		rpcEngBuilder, err := rpc.NewBuilder(suite.log, suite.state, rpc.Config{}, nil, nil, all.Blocks, all.Headers, collections, transactions, receipts,
 			results, suite.chainID, metrics, metrics, 0, 0, false, false, nil, nil)
 		require.NoError(suite.T(), err)
 		rpcEng, err := rpcEngBuilder.WithLegacy().Build()
 		require.NoError(suite.T(), err)
 
 		// create the ingest engine
-		ingestEng, err := ingestion.New(suite.log, suite.net, suite.state, suite.me, suite.request, blocks, headers, collections,
+		ingestEng, err := ingestion.New(suite.log, suite.net, suite.state, suite.me, suite.request, all.Blocks, all.Headers, collections,
 			transactions, results, receipts, metrics, collectionsToMarkFinalized, collectionsToMarkExecuted, blocksToMarkExecuted, rpcEng)
 		require.NoError(suite.T(), err)
 
 		// 1. Assume that follower engine updated the block storage and the protocol state. The block is reported as sealed
-		err = blocks.Store(&block)
+		err = all.Blocks.Store(&block)
 		require.NoError(suite.T(), err)
 		suite.snapshot.On("Head").Return(block.Header, nil).Twice()
 
@@ -683,11 +681,11 @@ func (suite *Suite) TestGetSealedTransaction() {
 // the correct block id
 func (suite *Suite) TestExecuteScript() {
 	unittest.RunWithBadgerDB(suite.T(), func(db *badger.DB) {
-		headers, _, _, _, _, blocks, _, _, _, _, _ := util.StorageLayer(suite.T(), db)
-		transactions := storage.NewTransactions(suite.metrics, db)
-		collections := storage.NewCollections(db, transactions)
-		results := storage.NewExecutionResults(suite.metrics, db)
-		receipts := storage.NewExecutionReceipts(suite.metrics, db, results, storage.DefaultCacheSize)
+		all := util.StorageLayer(suite.T(), db)
+		transactions := bstorage.NewTransactions(suite.metrics, db)
+		collections := bstorage.NewCollections(db, transactions)
+		results := bstorage.NewExecutionResults(suite.metrics, db)
+		receipts := bstorage.NewExecutionReceipts(suite.metrics, db, results, bstorage.DefaultCacheSize)
 
 		identities := unittest.IdentityListFixture(2, unittest.WithRole(flow.RoleExecution))
 		suite.snapshot.On("Identities", mock.Anything).Return(identities, nil)
@@ -699,8 +697,8 @@ func (suite *Suite) TestExecuteScript() {
 		suite.backend = backend.New(suite.state,
 			suite.collClient,
 			nil,
-			blocks,
-			headers,
+			all.Blocks,
+			all.Headers,
 			collections,
 			transactions,
 			receipts,
@@ -731,14 +729,14 @@ func (suite *Suite) TestExecuteScript() {
 		suite.net.On("Register", channels.ReceiveReceipts, mock.Anything).Return(conduit, nil).
 			Once()
 		// create the ingest engine
-		ingestEng, err := ingestion.New(suite.log, suite.net, suite.state, suite.me, suite.request, blocks, headers, collections,
+		ingestEng, err := ingestion.New(suite.log, suite.net, suite.state, suite.me, suite.request, all.Blocks, all.Headers, collections,
 			transactions, results, receipts, metrics, collectionsToMarkFinalized, collectionsToMarkExecuted, blocksToMarkExecuted, nil)
 		require.NoError(suite.T(), err)
 
 		// create a block and a seal pointing to that block
 		lastBlock := unittest.BlockFixture()
 		lastBlock.Header.Height = 2
-		err = blocks.Store(&lastBlock)
+		err = all.Blocks.Store(&lastBlock)
 		require.NoError(suite.T(), err)
 		err = db.Update(operation.IndexBlockHeight(lastBlock.Header.Height, lastBlock.ID()))
 		require.NoError(suite.T(), err)
@@ -755,7 +753,7 @@ func (suite *Suite) TestExecuteScript() {
 		// create another block as a predecessor of the block created earlier
 		prevBlock := unittest.BlockFixture()
 		prevBlock.Header.Height = lastBlock.Header.Height - 1
-		err = blocks.Store(&prevBlock)
+		err = all.Blocks.Store(&prevBlock)
 		require.NoError(suite.T(), err)
 		err = db.Update(operation.IndexBlockHeight(prevBlock.Header.Height, prevBlock.ID()))
 		require.NoError(suite.T(), err)

engine/common/follower/integration_test.go

@@ -49,17 +49,17 @@ func TestFollowerHappyPath(t *testing.T) {
 		metrics := metrics.NewNoopCollector()
 		tracer := trace.NewNoopTracer()
 		consumer := events.NewNoop()
-		headers, _, seals, index, payloads, blocks, qcs, setups, commits, statuses, results := storageutil.StorageLayer(t, db)
+		all := storageutil.StorageLayer(t, db)
 
 		// bootstrap root snapshot
-		state, err := pbadger.Bootstrap(metrics, db, headers, seals, results, blocks, qcs, setups, commits, statuses, rootSnapshot)
+		state, err := pbadger.Bootstrap(metrics, db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot)
 		require.NoError(t, err)
 		mockTimer := util.MockBlockTimer()
 
 		// create follower state
-		followerState, err := pbadger.NewFollowerState(state, index, payloads, tracer, consumer, mockTimer)
+		followerState, err := pbadger.NewFollowerState(state, all.Index, all.Payloads, tracer, consumer, mockTimer)
 		require.NoError(t, err)
-		finalizer := moduleconsensus.NewFinalizer(db, headers, followerState, tracer)
+		finalizer := moduleconsensus.NewFinalizer(db, all.Headers, followerState, tracer)
 		rootHeader, err := rootSnapshot.Head()
 		require.NoError(t, err)
 		rootQC, err := rootSnapshot.QuorumCertificate()
@@ -76,7 +76,7 @@ func TestFollowerHappyPath(t *testing.T) {
 
 		consensusConsumer := pubsub.NewFinalizationDistributor()
 		// use real consensus modules
-		forks, err := consensus.NewForks(rootHeader, headers, finalizer, consensusConsumer, rootHeader, rootQC)
+		forks, err := consensus.NewForks(rootHeader, all.Headers, finalizer, consensusConsumer, rootHeader, rootQC)
 		require.NoError(t, err)
 
 		// assume all proposals are valid
@@ -114,7 +114,7 @@ func TestFollowerHappyPath(t *testing.T) {
 		net.On("Register", mock.Anything, mock.Anything).Return(con, nil)
 
 		// use real engine
-		engine, err := NewComplianceLayer(unittest.Logger(), net, me, metrics, headers, rootHeader, followerCore)
+		engine, err := NewComplianceLayer(unittest.Logger(), net, me, metrics, all.Headers, rootHeader, followerCore)
 		require.NoError(t, err)
 		// don't forget to subscribe for finalization notifications
 		consensusConsumer.AddOnBlockFinalizedConsumer(engine.OnFinalizedBlock)

fvm/environment/contract_updater.go

@@ -57,9 +57,8 @@ func (lists *sortableContractUpdates) Less(i, j int) bool {
 	}
 }
 
-// ContractUpdater handles all smart contracts modification. It also captures
-// all changes as deltas and only commit them when called so smart contract
-// updates can be delayed until end of the tx execution.
+// ContractUpdater handles all smart contracts modification. It captures
+// contract updates and defer the updates to the end of the txn execution.
 //
 // Note that scripts cannot modify smart contracts, but must expose the API in
 // compliance with the runtime environment interface.

fvm/environment/derived_data_invalidator_test.go

@@ -257,8 +257,9 @@ func TestMeterParamOverridesUpdated(t *testing.T) {
 		snapshotTree)
 	require.NoError(t, err)
 
-	view := delta.NewDeltaView(snapshotTree.Append(executionSnapshot))
-	nestedTxn := state.NewTransactionState(view, state.DefaultParameters())
+	nestedTxn := state.NewTransactionState(
+		delta.NewDeltaView(snapshotTree.Append(executionSnapshot)),
+		state.DefaultParameters())
 
 	derivedBlockData := derived.NewEmptyDerivedBlockData()
 	derivedTxnData, err := derivedBlockData.NewDerivedTransactionData(0, 0)
@@ -300,7 +301,10 @@ func TestMeterParamOverridesUpdated(t *testing.T) {
 		require.Equal(t, expected, invalidator.MeterParamOverridesUpdated)
 	}
 
-	for _, registerId := range view.Finalize().AllRegisterIDs() {
+	executionSnapshot, err = nestedTxn.FinalizeMainTransaction()
+	require.NoError(t, err)
+
+	for _, registerId := range executionSnapshot.AllRegisterIDs() {
 		checkForUpdates(registerId, true)
 		checkForUpdates(
 			flow.NewRegisterID("other owner", registerId.Key),

fvm/environment/programs_test.go

@@ -89,14 +89,13 @@ var (
 )
 
 func setupProgramsTest(t *testing.T) storage.SnapshotTree {
-	view := delta.NewDeltaView(nil)
+	txnState := storage.SerialTransaction{
+		NestedTransaction: state.NewTransactionState(
+			delta.NewDeltaView(nil),
+			state.DefaultParameters()),
+	}
 
-	accounts := environment.NewAccounts(
-		storage.SerialTransaction{
-			NestedTransaction: state.NewTransactionState(
-				view,
-				state.DefaultParameters()),
-		})
+	accounts := environment.NewAccounts(txnState)
 
 	err := accounts.Create(nil, addressA)
 	require.NoError(t, err)
@@ -107,7 +106,10 @@ func setupProgramsTest(t *testing.T) storage.SnapshotTree {
 	err = accounts.Create(nil, addressC)
 	require.NoError(t, err)
 
-	return storage.NewSnapshotTree(nil).Append(view.Finalize())
+	executionSnapshot, err := txnState.FinalizeMainTransaction()
+	require.NoError(t, err)
+
+	return storage.NewSnapshotTree(nil).Append(executionSnapshot)
 }
 
 func getTestContract(
@@ -261,7 +263,7 @@ func Test_Programs(t *testing.T) {
 
 		require.Contains(t, output.Logs, "\"hello from A\"")
 
-		// same transaction should produce the exact same views
+		// same transaction should produce the exact same execution snapshots
 		// but only because we don't do any conditional update in a tx
 		compareExecutionSnapshots(t, executionSnapshotA, executionSnapshotA2)
 	})
@@ -338,7 +340,7 @@ func Test_Programs(t *testing.T) {
 		// rerun transaction
 
 		// execute transaction again, this time make sure it doesn't load code
-		execB2Snapshot := delta.NewDeltaView(state.NewReadFuncStorageSnapshot(
+		execB2Snapshot := state.NewReadFuncStorageSnapshot(
 			func(id flow.RegisterID) (flow.RegisterValue, error) {
 				idA := flow.ContractRegisterID(
 					flow.BytesToAddress([]byte(id.Owner)),
@@ -351,7 +353,7 @@ func Test_Programs(t *testing.T) {
 				require.NotEqual(t, id.Key, idB.Key)
 
 				return mainSnapshot.Get(id)
-			}))
+			})
 
 		executionSnapshotB2, output, err := vm.RunV2(
 			context,
@@ -779,14 +781,6 @@ func updateContractTx(name, code string, address flow.Address) *flow.Transaction
 	).AddAuthorizer(address)
 }
 
-// compareViews compares views using only data that matters (ie. two different hasher instances
-// trips the library comparison, even if actual SPoCKs are the same)
-func compareViews(t *testing.T, a, b *delta.View) {
-	require.Equal(t, a.Delta(), b.Delta())
-	require.Equal(t, a.Interactions(), b.Interactions())
-	require.Equal(t, a.SpockSecret(), b.SpockSecret())
-}
-
 func compareExecutionSnapshots(t *testing.T, a, b *state.ExecutionSnapshot) {
 	require.Equal(t, a.WriteSet, b.WriteSet)
 	require.Equal(t, a.ReadSet, b.ReadSet)