Merge #4168 #4179

4168: refactor logProgress r=zhangchiqing a=zhangchiqing

This PR:
1. Remove duplicated implementation of logProgress
2. although builds a lookup, the implementation is simplier
3. Also log 0% and 100%(before it only log from 10% to 90%). logging 0% is useful so that we know the processing is started. 100% is useful to know it's completed.

4179: mv fvm/derived fvm/storage/derived r=pattyshack a=pattyshack



Co-authored-by: Leo Zhang (zhangchiqing) <[email protected]>
Co-authored-by: Patrick Lee <[email protected]>

Author: 3 people

cmd/execution_config.go

@@ -18,7 +18,7 @@ import (
 
 	"github.com/onflow/flow-go/engine/execution/computation"
 	"github.com/onflow/flow-go/engine/execution/rpc"
-	"github.com/onflow/flow-go/fvm/derived"
+	"github.com/onflow/flow-go/fvm/storage/derived"
 	storage "github.com/onflow/flow-go/storage/badger"
 )
 

engine/execution/computation/computer/computer.go

@@ -15,9 +15,9 @@ import (
 	"github.com/onflow/flow-go/engine/execution/utils"
 	"github.com/onflow/flow-go/fvm"
 	"github.com/onflow/flow-go/fvm/blueprints"
-	"github.com/onflow/flow-go/fvm/derived"
 	"github.com/onflow/flow-go/fvm/state"
 	"github.com/onflow/flow-go/fvm/storage"
+	"github.com/onflow/flow-go/fvm/storage/derived"
 	"github.com/onflow/flow-go/model/flow"
 	"github.com/onflow/flow-go/module"
 	"github.com/onflow/flow-go/module/executiondatasync/provider"

engine/execution/computation/computer/computer_test.go

@@ -29,13 +29,13 @@ import (
 	"github.com/onflow/flow-go/engine/execution/state/delta"
 	"github.com/onflow/flow-go/engine/execution/testutil"
 	"github.com/onflow/flow-go/fvm"
-	"github.com/onflow/flow-go/fvm/derived"
 	"github.com/onflow/flow-go/fvm/environment"
 	fvmErrors "github.com/onflow/flow-go/fvm/errors"
 	fvmmock "github.com/onflow/flow-go/fvm/mock"
 	reusableRuntime "github.com/onflow/flow-go/fvm/runtime"
 	"github.com/onflow/flow-go/fvm/state"
 	"github.com/onflow/flow-go/fvm/storage"
+	"github.com/onflow/flow-go/fvm/storage/derived"
 	"github.com/onflow/flow-go/fvm/systemcontracts"
 	"github.com/onflow/flow-go/ledger"
 	"github.com/onflow/flow-go/model/flow"

engine/execution/computation/computer/mock/block_computer.go

@@ -28,9 +28,9 @@ import (
 	"github.com/onflow/flow-go/engine/verification/fetcher"
 	"github.com/onflow/flow-go/fvm"
 	"github.com/onflow/flow-go/fvm/blueprints"
-	"github.com/onflow/flow-go/fvm/derived"
 	"github.com/onflow/flow-go/fvm/environment"
 	"github.com/onflow/flow-go/fvm/errors"
+	"github.com/onflow/flow-go/fvm/storage/derived"
 	completeLedger "github.com/onflow/flow-go/ledger/complete"
 	"github.com/onflow/flow-go/ledger/complete/wal/fixtures"
 	"github.com/onflow/flow-go/model/flow"

engine/execution/computation/execution_verification_test.go

@@ -11,9 +11,9 @@ import (
 	"github.com/onflow/flow-go/engine/execution/computation/computer"
 	"github.com/onflow/flow-go/engine/execution/computation/query"
 	"github.com/onflow/flow-go/fvm"
-	"github.com/onflow/flow-go/fvm/derived"
 	reusableRuntime "github.com/onflow/flow-go/fvm/runtime"
 	"github.com/onflow/flow-go/fvm/state"
+	"github.com/onflow/flow-go/fvm/storage/derived"
 	"github.com/onflow/flow-go/model/flow"
 	"github.com/onflow/flow-go/module"
 	"github.com/onflow/flow-go/module/executiondatasync/provider"
@@ -217,7 +217,6 @@ func (e *Manager) ExecuteScript(
 		code,
 		arguments,
 		blockHeader,
-		e.derivedChainData.NewDerivedBlockDataForScript(blockHeader.ID()),
 		snapshot)
 }
 

engine/execution/computation/manager.go

@@ -19,9 +19,9 @@ import (
 	"github.com/onflow/flow-go/engine/execution/computation/computer"
 	"github.com/onflow/flow-go/engine/execution/testutil"
 	"github.com/onflow/flow-go/fvm"
-	"github.com/onflow/flow-go/fvm/derived"
 	reusableRuntime "github.com/onflow/flow-go/fvm/runtime"
 	"github.com/onflow/flow-go/fvm/storage"
+	"github.com/onflow/flow-go/fvm/storage/derived"
 	"github.com/onflow/flow-go/model/flow"
 	"github.com/onflow/flow-go/module/executiondatasync/execution_data"
 	exedataprovider "github.com/onflow/flow-go/module/executiondatasync/provider"
@@ -82,22 +82,17 @@ func mustFundAccounts(
 	execCtx fvm.Context,
 	accs *testAccounts,
 ) storage.SnapshotTree {
-	derivedBlockData := derived.NewEmptyDerivedBlockData()
-	execCtx = fvm.NewContextFromParent(
-		execCtx,
-		fvm.WithDerivedBlockData(derivedBlockData))
-
 	var err error
 	for _, acc := range accs.accounts {
 		transferTx := testutil.CreateTokenTransferTransaction(chain, 1_000_000, acc.address, chain.ServiceAddress())
 		err = testutil.SignTransactionAsServiceAccount(transferTx, accs.seq, chain)
 		require.NoError(b, err)
 		accs.seq++
 
-		tx := fvm.Transaction(
-			transferTx,
-			derivedBlockData.NextTxIndexForTestingOnly())
-		executionSnapshot, output, err := vm.Run(execCtx, tx, snapshotTree)
+		executionSnapshot, output, err := vm.Run(
+			execCtx,
+			fvm.Transaction(transferTx, 0),
+			snapshotTree)
 		require.NoError(b, err)
 		require.NoError(b, output.Err)
 		snapshotTree = snapshotTree.Append(executionSnapshot)

engine/execution/computation/manager_benchmark_test.go

@@ -28,10 +28,10 @@ import (
 	unittest2 "github.com/onflow/flow-go/engine/execution/state/unittest"
 	"github.com/onflow/flow-go/engine/execution/testutil"
 	"github.com/onflow/flow-go/fvm"
-	"github.com/onflow/flow-go/fvm/derived"
 	"github.com/onflow/flow-go/fvm/environment"
 	fvmErrors "github.com/onflow/flow-go/fvm/errors"
 	"github.com/onflow/flow-go/fvm/state"
+	"github.com/onflow/flow-go/fvm/storage/derived"
 	"github.com/onflow/flow-go/ledger/complete"
 	"github.com/onflow/flow-go/ledger/complete/wal/fixtures"
 	"github.com/onflow/flow-go/model/flow"

engine/execution/computation/manager_test.go

@@ -21,8 +21,8 @@ import (
 	"github.com/onflow/flow-go/engine/execution/computation/computer"
 	"github.com/onflow/flow-go/engine/execution/testutil"
 	"github.com/onflow/flow-go/fvm"
-	"github.com/onflow/flow-go/fvm/derived"
 	"github.com/onflow/flow-go/fvm/storage"
+	"github.com/onflow/flow-go/fvm/storage/derived"
 	"github.com/onflow/flow-go/model/flow"
 	"github.com/onflow/flow-go/module/executiondatasync/execution_data"
 	"github.com/onflow/flow-go/module/executiondatasync/provider"

engine/execution/computation/programs_test.go

@@ -13,8 +13,8 @@ import (
 	"github.com/rs/zerolog"
 
 	"github.com/onflow/flow-go/fvm"
-	"github.com/onflow/flow-go/fvm/derived"
 	"github.com/onflow/flow-go/fvm/state"
+	"github.com/onflow/flow-go/fvm/storage/derived"
 	"github.com/onflow/flow-go/model/flow"
 	"github.com/onflow/flow-go/module"
 	"github.com/onflow/flow-go/utils/debug"
@@ -32,7 +32,6 @@ type Executor interface {
 		script []byte,
 		arguments [][]byte,
 		blockHeader *flow.Header,
-		derivedBlockData *derived.DerivedBlockData,
 		snapshot state.StorageSnapshot,
 	) (
 		[]byte,
@@ -102,7 +101,6 @@ func (e *QueryExecutor) ExecuteScript(
 	script []byte,
 	arguments [][]byte,
 	blockHeader *flow.Header,
-	derivedBlockData *derived.DerivedBlockData,
 	snapshot state.StorageSnapshot,
 ) (
 	encodedValue []byte,
@@ -163,7 +161,8 @@ func (e *QueryExecutor) ExecuteScript(
 		fvm.NewContextFromParent(
 			e.vmCtx,
 			fvm.WithBlockHeader(blockHeader),
-			fvm.WithDerivedBlockData(derivedBlockData)),
+			fvm.WithDerivedBlockData(
+				e.derivedChainData.NewDerivedBlockDataForScript(blockHeader.ID()))),
 		fvm.NewScriptWithContextAndArgs(script, requestCtx, arguments...),
 		snapshot)
 	if err != nil {