Skip to content

Commit 503717a

Browse files
bors[bot]janezpodhostnik
bors[bot]
and
janezpodhostnik
authored
Merge #4140
4140: Storage layer test refactor r=janezpodhostnik a=janezpodhostnik Extracting pieces from #3736 so its easier to digest and merge. Co-authored-by: Janez Podhostnik <[email protected]>
2 parents 425675f + 52c46c8 commit 503717a

File tree

9 files changed

+124
-152
lines changed

9 files changed

+124
-152
lines changed

‎engine/access/access_test.go‎

Lines changed: 41 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,8 @@ import (
3939
"github.com/onflow/flow-go/network/channels"
4040
"github.com/onflow/flow-go/network/mocknetwork"
4141
protocol "github.com/onflow/flow-go/state/protocol/mock"
42-
storage "github.com/onflow/flow-go/storage/badger"
42+
"github.com/onflow/flow-go/storage"
43+
bstorage "github.com/onflow/flow-go/storage/badger"
4344
"github.com/onflow/flow-go/storage/badger/operation"
4445
"github.com/onflow/flow-go/storage/util"
4546
"github.com/onflow/flow-go/utils/unittest"
@@ -109,23 +110,20 @@ func (suite *Suite) SetupTest() {
109110
}
110111

111112
func (suite *Suite) RunTest(
112-
f func(handler *access.Handler, db *badger.DB, blocks *storage.Blocks, headers *storage.Headers, results *storage.ExecutionResults),
113+
f func(handler *access.Handler, db *badger.DB, all *storage.All),
113114
) {
114115
unittest.RunWithBadgerDB(suite.T(), func(db *badger.DB) {
115-
headers, _, _, _, _, blocks, _, _, _, _, results := util.StorageLayer(suite.T(), db)
116-
transactions := storage.NewTransactions(suite.metrics, db)
117-
collections := storage.NewCollections(db, transactions)
118-
receipts := storage.NewExecutionReceipts(suite.metrics, db, results, storage.DefaultCacheSize)
116+
all := util.StorageLayer(suite.T(), db)
119117

120118
suite.backend = backend.New(suite.state,
121119
suite.collClient,
122120
nil,
123-
blocks,
124-
headers,
125-
collections,
126-
transactions,
127-
receipts,
128-
results,
121+
all.Blocks,
122+
all.Headers,
123+
all.Collections,
124+
all.Transactions,
125+
all.Receipts,
126+
all.Results,
129127
suite.chainID,
130128
suite.metrics,
131129
nil,
@@ -138,12 +136,12 @@ func (suite *Suite) RunTest(
138136
)
139137

140138
handler := access.NewHandler(suite.backend, suite.chainID.Chain(), access.WithBlockSignerDecoder(suite.signerIndicesDecoder))
141-
f(handler, db, blocks, headers, results)
139+
f(handler, db, all)
142140
})
143141
}
144142

145143
func (suite *Suite) TestSendAndGetTransaction() {
146-
suite.RunTest(func(handler *access.Handler, _ *badger.DB, _ *storage.Blocks, _ *storage.Headers, _ *storage.ExecutionResults) {
144+
suite.RunTest(func(handler *access.Handler, _ *badger.DB, _ *storage.All) {
147145
referenceBlock := unittest.BlockHeaderFixture()
148146
transaction := unittest.TransactionFixture()
149147
transaction.SetReferenceBlockID(referenceBlock.ID())
@@ -196,7 +194,7 @@ func (suite *Suite) TestSendAndGetTransaction() {
196194
}
197195

198196
func (suite *Suite) TestSendExpiredTransaction() {
199-
suite.RunTest(func(handler *access.Handler, _ *badger.DB, _ *storage.Blocks, _ *storage.Headers, _ *storage.ExecutionResults) {
197+
suite.RunTest(func(handler *access.Handler, _ *badger.DB, _ *storage.All) {
200198
referenceBlock := unittest.BlockHeaderFixture()
201199

202200
// create latest block that is past the expiry window
@@ -251,8 +249,8 @@ func (suite *Suite) TestSendTransactionToRandomCollectionNode() {
251249

252250
// create storage
253251
metrics := metrics.NewNoopCollector()
254-
transactions := storage.NewTransactions(metrics, db)
255-
collections := storage.NewCollections(db, transactions)
252+
transactions := bstorage.NewTransactions(metrics, db)
253+
collections := bstorage.NewCollections(db, transactions)
256254

257255
// create collection node cluster
258256
count := 2
@@ -349,16 +347,16 @@ func (suite *Suite) TestSendTransactionToRandomCollectionNode() {
349347
}
350348

351349
func (suite *Suite) TestGetBlockByIDAndHeight() {
352-
suite.RunTest(func(handler *access.Handler, db *badger.DB, blocks *storage.Blocks, _ *storage.Headers, _ *storage.ExecutionResults) {
350+
suite.RunTest(func(handler *access.Handler, db *badger.DB, all *storage.All) {
353351

354352
// test block1 get by ID
355353
block1 := unittest.BlockFixture()
356354
// test block2 get by height
357355
block2 := unittest.BlockFixture()
358356
block2.Header.Height = 2
359357

360-
require.NoError(suite.T(), blocks.Store(&block1))
361-
require.NoError(suite.T(), blocks.Store(&block2))
358+
require.NoError(suite.T(), all.Blocks.Store(&block1))
359+
require.NoError(suite.T(), all.Blocks.Store(&block2))
362360

363361
// the follower logic should update height index on the block storage when a block is finalized
364362
err := db.Update(operation.IndexBlockHeight(block2.Header.Height, block2.ID()))
@@ -473,7 +471,7 @@ func (suite *Suite) TestGetBlockByIDAndHeight() {
473471
}
474472

475473
func (suite *Suite) TestGetExecutionResultByBlockID() {
476-
suite.RunTest(func(handler *access.Handler, db *badger.DB, blocks *storage.Blocks, _ *storage.Headers, executionResults *storage.ExecutionResults) {
474+
suite.RunTest(func(handler *access.Handler, db *badger.DB, all *storage.All) {
477475

478476
// test block1 get by ID
479477
nonexistingID := unittest.IdentifierFixture()
@@ -483,8 +481,8 @@ func (suite *Suite) TestGetExecutionResultByBlockID() {
483481
unittest.WithExecutionResultBlockID(blockID),
484482
unittest.WithServiceEvents(2))
485483

486-
require.NoError(suite.T(), executionResults.Store(er))
487-
require.NoError(suite.T(), executionResults.Index(blockID, er.ID()))
484+
require.NoError(suite.T(), all.Results.Store(er))
485+
require.NoError(suite.T(), all.Results.Index(blockID, er.ID()))
488486

489487
assertResp := func(resp *accessproto.ExecutionResultForBlockIDResponse, err error, executionResult *flow.ExecutionResult) {
490488
require.NoError(suite.T(), err)
@@ -555,9 +553,9 @@ func (suite *Suite) TestGetExecutionResultByBlockID() {
555553
// is reported as sealed
556554
func (suite *Suite) TestGetSealedTransaction() {
557555
unittest.RunWithBadgerDB(suite.T(), func(db *badger.DB) {
558-
headers, _, _, _, _, blocks, _, _, _, _, _ := util.StorageLayer(suite.T(), db)
559-
results := storage.NewExecutionResults(suite.metrics, db)
560-
receipts := storage.NewExecutionReceipts(suite.metrics, db, results, storage.DefaultCacheSize)
556+
all := util.StorageLayer(suite.T(), db)
557+
results := bstorage.NewExecutionResults(suite.metrics, db)
558+
receipts := bstorage.NewExecutionReceipts(suite.metrics, db, results, bstorage.DefaultCacheSize)
561559
enIdentities := unittest.IdentityListFixture(2, unittest.WithRole(flow.RoleExecution))
562560
enNodeIDs := flow.IdentifierList(enIdentities.NodeIDs())
563561

@@ -594,8 +592,8 @@ func (suite *Suite) TestGetSealedTransaction() {
594592

595593
// initialize storage
596594
metrics := metrics.NewNoopCollector()
597-
transactions := storage.NewTransactions(metrics, db)
598-
collections := storage.NewCollections(db, transactions)
595+
transactions := bstorage.NewTransactions(metrics, db)
596+
collections := bstorage.NewCollections(db, transactions)
599597
collectionsToMarkFinalized, err := stdmap.NewTimes(100)
600598
require.NoError(suite.T(), err)
601599
collectionsToMarkExecuted, err := stdmap.NewTimes(100)
@@ -606,8 +604,8 @@ func (suite *Suite) TestGetSealedTransaction() {
606604
backend := backend.New(suite.state,
607605
suite.collClient,
608606
nil,
609-
blocks,
610-
headers,
607+
all.Blocks,
608+
all.Headers,
611609
collections,
612610
transactions,
613611
receipts,
@@ -625,19 +623,19 @@ func (suite *Suite) TestGetSealedTransaction() {
625623

626624
handler := access.NewHandler(backend, suite.chainID.Chain())
627625

628-
rpcEngBuilder, err := rpc.NewBuilder(suite.log, suite.state, rpc.Config{}, nil, nil, blocks, headers, collections, transactions, receipts,
626+
rpcEngBuilder, err := rpc.NewBuilder(suite.log, suite.state, rpc.Config{}, nil, nil, all.Blocks, all.Headers, collections, transactions, receipts,
629627
results, suite.chainID, metrics, metrics, 0, 0, false, false, nil, nil)
630628
require.NoError(suite.T(), err)
631629
rpcEng, err := rpcEngBuilder.WithLegacy().Build()
632630
require.NoError(suite.T(), err)
633631

634632
// create the ingest engine
635-
ingestEng, err := ingestion.New(suite.log, suite.net, suite.state, suite.me, suite.request, blocks, headers, collections,
633+
ingestEng, err := ingestion.New(suite.log, suite.net, suite.state, suite.me, suite.request, all.Blocks, all.Headers, collections,
636634
transactions, results, receipts, metrics, collectionsToMarkFinalized, collectionsToMarkExecuted, blocksToMarkExecuted, rpcEng)
637635
require.NoError(suite.T(), err)
638636

639637
// 1. Assume that follower engine updated the block storage and the protocol state. The block is reported as sealed
640-
err = blocks.Store(&block)
638+
err = all.Blocks.Store(&block)
641639
require.NoError(suite.T(), err)
642640
suite.snapshot.On("Head").Return(block.Header, nil).Twice()
643641

@@ -683,11 +681,11 @@ func (suite *Suite) TestGetSealedTransaction() {
683681
// the correct block id
684682
func (suite *Suite) TestExecuteScript() {
685683
unittest.RunWithBadgerDB(suite.T(), func(db *badger.DB) {
686-
headers, _, _, _, _, blocks, _, _, _, _, _ := util.StorageLayer(suite.T(), db)
687-
transactions := storage.NewTransactions(suite.metrics, db)
688-
collections := storage.NewCollections(db, transactions)
689-
results := storage.NewExecutionResults(suite.metrics, db)
690-
receipts := storage.NewExecutionReceipts(suite.metrics, db, results, storage.DefaultCacheSize)
684+
all := util.StorageLayer(suite.T(), db)
685+
transactions := bstorage.NewTransactions(suite.metrics, db)
686+
collections := bstorage.NewCollections(db, transactions)
687+
results := bstorage.NewExecutionResults(suite.metrics, db)
688+
receipts := bstorage.NewExecutionReceipts(suite.metrics, db, results, bstorage.DefaultCacheSize)
691689

692690
identities := unittest.IdentityListFixture(2, unittest.WithRole(flow.RoleExecution))
693691
suite.snapshot.On("Identities", mock.Anything).Return(identities, nil)
@@ -699,8 +697,8 @@ func (suite *Suite) TestExecuteScript() {
699697
suite.backend = backend.New(suite.state,
700698
suite.collClient,
701699
nil,
702-
blocks,
703-
headers,
700+
all.Blocks,
701+
all.Headers,
704702
collections,
705703
transactions,
706704
receipts,
@@ -731,14 +729,14 @@ func (suite *Suite) TestExecuteScript() {
731729
suite.net.On("Register", channels.ReceiveReceipts, mock.Anything).Return(conduit, nil).
732730
Once()
733731
// create the ingest engine
734-
ingestEng, err := ingestion.New(suite.log, suite.net, suite.state, suite.me, suite.request, blocks, headers, collections,
732+
ingestEng, err := ingestion.New(suite.log, suite.net, suite.state, suite.me, suite.request, all.Blocks, all.Headers, collections,
735733
transactions, results, receipts, metrics, collectionsToMarkFinalized, collectionsToMarkExecuted, blocksToMarkExecuted, nil)
736734
require.NoError(suite.T(), err)
737735

738736
// create a block and a seal pointing to that block
739737
lastBlock := unittest.BlockFixture()
740738
lastBlock.Header.Height = 2
741-
err = blocks.Store(&lastBlock)
739+
err = all.Blocks.Store(&lastBlock)
742740
require.NoError(suite.T(), err)
743741
err = db.Update(operation.IndexBlockHeight(lastBlock.Header.Height, lastBlock.ID()))
744742
require.NoError(suite.T(), err)
@@ -755,7 +753,7 @@ func (suite *Suite) TestExecuteScript() {
755753
// create another block as a predecessor of the block created earlier
756754
prevBlock := unittest.BlockFixture()
757755
prevBlock.Header.Height = lastBlock.Header.Height - 1
758-
err = blocks.Store(&prevBlock)
756+
err = all.Blocks.Store(&prevBlock)
759757
require.NoError(suite.T(), err)
760758
err = db.Update(operation.IndexBlockHeight(prevBlock.Header.Height, prevBlock.ID()))
761759
require.NoError(suite.T(), err)

‎engine/common/follower/integration_test.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -49,17 +49,17 @@ func TestFollowerHappyPath(t *testing.T) {
4949
metrics := metrics.NewNoopCollector()
5050
tracer := trace.NewNoopTracer()
5151
consumer := events.NewNoop()
52-
headers, _, seals, index, payloads, blocks, qcs, setups, commits, statuses, results := storageutil.StorageLayer(t, db)
52+
all := storageutil.StorageLayer(t, db)
5353

5454
// bootstrap root snapshot
55-
state, err := pbadger.Bootstrap(metrics, db, headers, seals, results, blocks, qcs, setups, commits, statuses, rootSnapshot)
55+
state, err := pbadger.Bootstrap(metrics, db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot)
5656
require.NoError(t, err)
5757
mockTimer := util.MockBlockTimer()
5858

5959
// create follower state
60-
followerState, err := pbadger.NewFollowerState(state, index, payloads, tracer, consumer, mockTimer)
60+
followerState, err := pbadger.NewFollowerState(state, all.Index, all.Payloads, tracer, consumer, mockTimer)
6161
require.NoError(t, err)
62-
finalizer := moduleconsensus.NewFinalizer(db, headers, followerState, tracer)
62+
finalizer := moduleconsensus.NewFinalizer(db, all.Headers, followerState, tracer)
6363
rootHeader, err := rootSnapshot.Head()
6464
require.NoError(t, err)
6565
rootQC, err := rootSnapshot.QuorumCertificate()
@@ -76,7 +76,7 @@ func TestFollowerHappyPath(t *testing.T) {
7676

7777
consensusConsumer := pubsub.NewFinalizationDistributor()
7878
// use real consensus modules
79-
forks, err := consensus.NewForks(rootHeader, headers, finalizer, consensusConsumer, rootHeader, rootQC)
79+
forks, err := consensus.NewForks(rootHeader, all.Headers, finalizer, consensusConsumer, rootHeader, rootQC)
8080
require.NoError(t, err)
8181

8282
// assume all proposals are valid
@@ -114,7 +114,7 @@ func TestFollowerHappyPath(t *testing.T) {
114114
net.On("Register", mock.Anything, mock.Anything).Return(con, nil)
115115

116116
// use real engine
117-
engine, err := NewComplianceLayer(unittest.Logger(), net, me, metrics, headers, rootHeader, followerCore)
117+
engine, err := NewComplianceLayer(unittest.Logger(), net, me, metrics, all.Headers, rootHeader, followerCore)
118118
require.NoError(t, err)
119119
// don't forget to subscribe for finalization notifications
120120
consensusConsumer.AddOnBlockFinalizedConsumer(engine.OnFinalizedBlock)

‎module/builder/collection/builder_test.go‎

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,8 @@ import (
2626
"github.com/onflow/flow-go/state/protocol/events"
2727
"github.com/onflow/flow-go/state/protocol/inmem"
2828
"github.com/onflow/flow-go/state/protocol/util"
29-
storage "github.com/onflow/flow-go/storage/badger"
29+
"github.com/onflow/flow-go/storage"
30+
bstorage "github.com/onflow/flow-go/storage/badger"
3031
"github.com/onflow/flow-go/storage/badger/operation"
3132
"github.com/onflow/flow-go/storage/badger/procedure"
3233
sutil "github.com/onflow/flow-go/storage/util"
@@ -43,9 +44,9 @@ type BuilderSuite struct {
4344
genesis *model.Block
4445
chainID flow.ChainID
4546

46-
headers *storage.Headers
47-
payloads *storage.ClusterPayloads
48-
blocks *storage.Blocks
47+
headers storage.Headers
48+
payloads storage.ClusterPayloads
49+
blocks storage.Blocks
4950

5051
state cluster.MutableState
5152

@@ -73,11 +74,11 @@ func (suite *BuilderSuite) SetupTest() {
7374

7475
metrics := metrics.NewNoopCollector()
7576
tracer := trace.NewNoopTracer()
76-
headers, _, seals, index, conPayloads, blocks, qcs, setups, commits, statuses, results := sutil.StorageLayer(suite.T(), suite.db)
77+
all := sutil.StorageLayer(suite.T(), suite.db)
7778
consumer := events.NewNoop()
78-
suite.headers = headers
79-
suite.blocks = blocks
80-
suite.payloads = storage.NewClusterPayloads(metrics, suite.db)
79+
suite.headers = all.Headers
80+
suite.blocks = all.Blocks
81+
suite.payloads = bstorage.NewClusterPayloads(metrics, suite.db)
8182

8283
clusterQC := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(suite.genesis.ID()))
8384
clusterStateRoot, err := clusterkv.NewStateRoot(suite.genesis, clusterQC)
@@ -98,10 +99,10 @@ func (suite *BuilderSuite) SetupTest() {
9899
rootSnapshot, err := inmem.SnapshotFromBootstrapState(root, result, seal, unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(root.ID())))
99100
require.NoError(suite.T(), err)
100101

101-
state, err := pbadger.Bootstrap(metrics, suite.db, headers, seals, results, blocks, qcs, setups, commits, statuses, rootSnapshot)
102+
state, err := pbadger.Bootstrap(metrics, suite.db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot)
102103
require.NoError(suite.T(), err)
103104

104-
suite.protoState, err = pbadger.NewFollowerState(state, index, conPayloads, tracer, consumer, util.MockBlockTimer())
105+
suite.protoState, err = pbadger.NewFollowerState(state, all.Index, all.Payloads, tracer, consumer, util.MockBlockTimer())
105106
require.NoError(suite.T(), err)
106107

107108
// add some transactions to transaction pool
@@ -979,10 +980,10 @@ func benchmarkBuildOn(b *testing.B, size int) {
979980

980981
metrics := metrics.NewNoopCollector()
981982
tracer := trace.NewNoopTracer()
982-
headers, _, _, _, _, blocks, _, _, _, _, _ := sutil.StorageLayer(suite.T(), suite.db)
983-
suite.headers = headers
984-
suite.blocks = blocks
985-
suite.payloads = storage.NewClusterPayloads(metrics, suite.db)
983+
all := sutil.StorageLayer(suite.T(), suite.db)
984+
suite.headers = all.Headers
985+
suite.blocks = all.Blocks
986+
suite.payloads = bstorage.NewClusterPayloads(metrics, suite.db)
986987

987988
qc := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(suite.genesis.ID()))
988989
stateRoot, err := clusterkv.NewStateRoot(suite.genesis, qc)

‎state/cluster/badger/mutator_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -62,14 +62,14 @@ func (suite *MutatorSuite) SetupTest() {
6262

6363
metrics := metrics.NewNoopCollector()
6464
tracer := trace.NewNoopTracer()
65-
headers, _, seals, index, conPayloads, blocks, qcs, setups, commits, statuses, results := util.StorageLayer(suite.T(), suite.db)
65+
all := util.StorageLayer(suite.T(), suite.db)
6666
colPayloads := storage.NewClusterPayloads(metrics, suite.db)
6767

6868
clusterStateRoot, err := NewStateRoot(suite.genesis, unittest.QuorumCertificateFixture())
6969
suite.NoError(err)
7070
clusterState, err := Bootstrap(suite.db, clusterStateRoot)
7171
suite.Assert().Nil(err)
72-
suite.state, err = NewMutableState(clusterState, tracer, headers, colPayloads)
72+
suite.state, err = NewMutableState(clusterState, tracer, all.Headers, colPayloads)
7373
suite.Assert().Nil(err)
7474
consumer := events.NewNoop()
7575

@@ -86,10 +86,10 @@ func (suite *MutatorSuite) SetupTest() {
8686

8787
suite.protoGenesis = genesis.Header
8888

89-
state, err := pbadger.Bootstrap(metrics, suite.db, headers, seals, results, blocks, qcs, setups, commits, statuses, rootSnapshot)
89+
state, err := pbadger.Bootstrap(metrics, suite.db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot)
9090
require.NoError(suite.T(), err)
9191

92-
suite.protoState, err = pbadger.NewFollowerState(state, index, conPayloads, tracer, consumer, protocolutil.MockBlockTimer())
92+
suite.protoState, err = pbadger.NewFollowerState(state, all.Index, all.Payloads, tracer, consumer, protocolutil.MockBlockTimer())
9393
require.NoError(suite.T(), err)
9494
}
9595

‎state/cluster/badger/snapshot_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,20 +55,20 @@ func (suite *SnapshotSuite) SetupTest() {
5555
metrics := metrics.NewNoopCollector()
5656
tracer := trace.NewNoopTracer()
5757

58-
headers, _, seals, _, _, blocks, qcs, setups, commits, statuses, results := util.StorageLayer(suite.T(), suite.db)
58+
all := util.StorageLayer(suite.T(), suite.db)
5959
colPayloads := storage.NewClusterPayloads(metrics, suite.db)
6060

6161
clusterStateRoot, err := NewStateRoot(suite.genesis, unittest.QuorumCertificateFixture())
6262
suite.Assert().Nil(err)
6363
clusterState, err := Bootstrap(suite.db, clusterStateRoot)
6464
suite.Assert().Nil(err)
65-
suite.state, err = NewMutableState(clusterState, tracer, headers, colPayloads)
65+
suite.state, err = NewMutableState(clusterState, tracer, all.Headers, colPayloads)
6666
suite.Assert().Nil(err)
6767

6868
participants := unittest.IdentityListFixture(5, unittest.WithAllRoles())
6969
root := unittest.RootSnapshotFixture(participants)
7070

71-
suite.protoState, err = pbadger.Bootstrap(metrics, suite.db, headers, seals, results, blocks, qcs, setups, commits, statuses, root)
71+
suite.protoState, err = pbadger.Bootstrap(metrics, suite.db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, root)
7272
require.NoError(suite.T(), err)
7373

7474
suite.Require().Nil(err)

0 commit comments

Comments
 (0)