Merge branch 'master' into jp/support-private-registry

Author: sjonpaulbrown

Makefile

@@ -859,6 +859,7 @@ docker-all-tools: tool-util tool-remove-execution-fork
 PHONY: docker-build-util
 docker-build-util:
 	docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/util --build-arg GOARCH=$(GOARCH) --build-arg VERSION=$(IMAGE_TAG) --build-arg CGO_FLAG=$(DISABLE_ADX) --target production \
+		--secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \
 		-t "$(CONTAINER_REGISTRY)/util:latest"  \
 		-t "$(CONTAINER_REGISTRY)/util:$(IMAGE_TAG)" .
 

cmd/util/cmd/export-evm-state/cmd.go

@@ -3,13 +3,17 @@ package evm_exporter
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 
+	"github.com/onflow/atree"
+
 	"github.com/onflow/flow-go/cmd/util/ledger/util"
 	"github.com/onflow/flow-go/fvm/evm"
 	"github.com/onflow/flow-go/fvm/evm/emulator/state"
+	"github.com/onflow/flow-go/fvm/evm/testutils"
 	"github.com/onflow/flow-go/ledger"
 	"github.com/onflow/flow-go/ledger/common/convert"
 	"github.com/onflow/flow-go/model/flow"
@@ -20,6 +24,8 @@ var (
 	flagExecutionStateDir string
 	flagOutputDir         string
 	flagStateCommitment   string
+	flagEVMStateGobDir    string
+	flagEVMStateGobHeight uint64
 )
 
 var Cmd = &cobra.Command{
@@ -34,21 +40,33 @@ func init() {
 
 	Cmd.Flags().StringVar(&flagExecutionStateDir, "execution-state-dir", "",
 		"Execution Node state dir (where WAL logs are written")
-	_ = Cmd.MarkFlagRequired("execution-state-dir")
 
 	Cmd.Flags().StringVar(&flagOutputDir, "output-dir", "",
 		"Directory to write new Execution State to")
 	_ = Cmd.MarkFlagRequired("output-dir")
 
 	Cmd.Flags().StringVar(&flagStateCommitment, "state-commitment", "",
 		"State commitment (hex-encoded, 64 characters)")
+
+	Cmd.Flags().StringVar(&flagEVMStateGobDir, "evm_state_gob_dir", "/var/flow/data/evm_state_gob",
+		"directory that stores the evm state gob files as checkpoint")
+
+	Cmd.Flags().Uint64Var(&flagEVMStateGobHeight, "evm_state_gob_height", 0,
+		"the flow height of the evm state gob files")
 }
 
 func run(*cobra.Command, []string) {
 	log.Info().Msg("start exporting evm state")
-	err := ExportEVMState(flagChain, flagExecutionStateDir, flagStateCommitment, flagOutputDir)
-	if err != nil {
-		log.Fatal().Err(err).Msg("cannot get export evm state")
+	if flagExecutionStateDir != "" {
+		err := ExportEVMState(flagChain, flagExecutionStateDir, flagStateCommitment, flagOutputDir)
+		if err != nil {
+			log.Fatal().Err(err).Msg("cannot get export evm state")
+		}
+	} else if flagEVMStateGobDir != "" {
+		err := ExportEVMStateFromGob(flagChain, flagEVMStateGobDir, flagEVMStateGobHeight, flagOutputDir)
+		if err != nil {
+			log.Fatal().Err(err).Msg("cannot get export evm state from gob files")
+		}
 	}
 }
 
@@ -83,7 +101,40 @@ func ExportEVMState(
 
 	payloadsLedger := util.NewPayloadsLedger(filteredPayloads)
 
-	exporter, err := state.NewExporter(payloadsLedger, storageRoot)
+	return ExportEVMStateFromPayloads(payloadsLedger, storageRoot, outputPath)
+}
+
+func ExportEVMStateFromGob(
+	chainName string,
+	evmStateGobDir string,
+	flowHeight uint64,
+	outputPath string) error {
+
+	valueFileName, allocatorFileName := evmStateGobFileNamesByEndHeight(evmStateGobDir, flowHeight)
+	chainID := flow.ChainID(chainName)
+
+	storageRoot := evm.StorageAccountAddress(chainID)
+	valuesGob, err := testutils.DeserializeState(valueFileName)
+	if err != nil {
+		return err
+	}
+
+	allocatorGobs, err := testutils.DeserializeAllocator(allocatorFileName)
+	if err != nil {
+		return err
+	}
+
+	store := testutils.GetSimpleValueStorePopulated(valuesGob, allocatorGobs)
+
+	return ExportEVMStateFromPayloads(store, storageRoot, outputPath)
+}
+
+func ExportEVMStateFromPayloads(
+	ledger atree.Ledger,
+	storageRoot flow.Address,
+	outputPath string,
+) error {
+	exporter, err := state.NewExporter(ledger, storageRoot)
 	if err != nil {
 		return fmt.Errorf("failed to create exporter: %w", err)
 	}
@@ -95,15 +146,15 @@ func ExportEVMState(
 		}
 	}
 
-	fi, err := os.Create(outputPath)
-	if err != nil {
-		return err
-	}
-	defer fi.Close()
-
-	err = exporter.Export(outputPath)
+	err = exporter.ExportGob(outputPath)
 	if err != nil {
 		return fmt.Errorf("failed to export: %w", err)
 	}
 	return nil
 }
+
+func evmStateGobFileNamesByEndHeight(evmStateGobDir string, endHeight uint64) (string, string) {
+	valueFileName := filepath.Join(evmStateGobDir, fmt.Sprintf("values-%d.gob", endHeight))
+	allocatorFileName := filepath.Join(evmStateGobDir, fmt.Sprintf("allocators-%d.gob", endHeight))
+	return valueFileName, allocatorFileName
+}

fvm/evm/emulator/state/base.go

@@ -74,13 +74,13 @@ func NewBaseView(ledger atree.Ledger, rootAddress flow.Address) (*BaseView, erro
 	// fetch the account collection, if not exist, create one
 	view.accounts, view.accountSetupOnCommit, err = view.fetchOrCreateCollection(AccountsStorageIDKey)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to fetch or create account collection with key %v: %w", AccountsStorageIDKey, err)
 	}
 
 	// fetch the code collection, if not exist, create one
 	view.codes, view.codeSetupOnCommit, err = view.fetchOrCreateCollection(CodesStorageIDKey)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to fetch or create code collection with key %v: %w", CodesStorageIDKey, err)
 	}
 
 	return view, nil
@@ -485,7 +485,10 @@ func (v *BaseView) fetchOrCreateCollection(path string) (collection *Collection,
 	}
 	if len(collectionID) == 0 {
 		collection, err = v.collectionProvider.NewCollection()
-		return collection, true, err
+		if err != nil {
+			return collection, true, fmt.Errorf("fail to create collection with key %v: %w", path, err)
+		}
+		return collection, true, nil
 	}
 	collection, err = v.collectionProvider.CollectionByID(collectionID)
 	return collection, false, err

fvm/evm/emulator/state/diff.go

@@ -0,0 +1,91 @@
+package state
+
+import (
+	"bytes"
+	"fmt"
+)
+
+func AccountEqual(a, b *Account) bool {
+	if a.Address != b.Address {
+		return false
+	}
+	if !bytes.Equal(a.Balance.Bytes(), b.Balance.Bytes()) {
+		return false
+	}
+	if a.Nonce != b.Nonce {
+		return false
+	}
+	if a.CodeHash != b.CodeHash {
+		return false
+	}
+
+	// CollectionID could be different
+	return true
+}
+
+// find the difference and return as error
+func Diff(a *EVMState, b *EVMState) []error {
+	var differences []error
+
+	// Compare Accounts
+	for addr, accA := range a.Accounts {
+		if accB, exists := b.Accounts[addr]; exists {
+			if !AccountEqual(accA, accB) {
+				differences = append(differences, fmt.Errorf("account %s differs, accA %v, accB %v", addr.Hex(), accA, accB))
+			}
+		} else {
+			differences = append(differences, fmt.Errorf("account %s exists in a but not in b", addr.Hex()))
+		}
+	}
+	for addr := range b.Accounts {
+		if _, exists := a.Accounts[addr]; !exists {
+			differences = append(differences, fmt.Errorf("account %s exists in b but not in a", addr.Hex()))
+		}
+	}
+
+	// Compare Slots
+	for addr, slotsA := range a.Slots {
+		slotsB, exists := b.Slots[addr]
+		if !exists {
+			differences = append(differences, fmt.Errorf("slots for address %s exist in a but not in b", addr.Hex()))
+			continue
+		}
+		for key, valueA := range slotsA {
+			if valueB, exists := slotsB[key]; exists {
+				if valueA.Value != valueB.Value {
+					differences = append(differences, fmt.Errorf("slot value for address %s and key %s differs", addr.Hex(), key.Hex()))
+				}
+			} else {
+				differences = append(differences, fmt.Errorf("slot with key %s for address %s exists in a but not in b", key.Hex(), addr.Hex()))
+			}
+		}
+		for key := range slotsB {
+			if _, exists := slotsA[key]; !exists {
+				differences = append(differences, fmt.Errorf("slot with key %s for address %s exists in b but not in a", key.Hex(), addr.Hex()))
+			}
+		}
+	}
+	for addr := range b.Slots {
+		if _, exists := a.Slots[addr]; !exists {
+			differences = append(differences, fmt.Errorf("slots for address %s exist in b but not in a", addr.Hex()))
+		}
+	}
+
+	// Compare Codes
+	for hash, codeA := range a.Codes {
+		if codeB, exists := b.Codes[hash]; exists {
+			if !bytes.Equal(codeA.Code, codeB.Code) {
+				differences = append(differences, fmt.Errorf("code for hash %s differs", hash.Hex()))
+			}
+		} else {
+			differences = append(differences, fmt.Errorf("code with hash %s exists in a but not in b", hash.Hex()))
+		}
+	}
+	for hash := range b.Codes {
+		if _, exists := a.Codes[hash]; !exists {
+			differences = append(differences, fmt.Errorf("code with hash %s exists in b but not in a", hash.Hex()))
+		}
+	}
+
+	return differences
+}

fvm/evm/emulator/state/diff_test.go

@@ -0,0 +1,74 @@
+package state_test
+
+import (
+	"fmt"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/onflow/flow-go/fvm/evm"
+	"github.com/onflow/flow-go/fvm/evm/emulator/state"
+	"github.com/onflow/flow-go/fvm/evm/testutils"
+	"github.com/onflow/flow-go/model/flow"
+)
+
+func StateDiff(t *testing.T) {
+	offchainState, err := state.ImportEVMStateFromGob("/var/flow2/evm-state-from-gobs-218215348/")
+	require.NoError(t, err)
+
+	enState, err := state.ImportEVMStateFromGob("/var/flow2/evm-state-from-gobs-218215348/")
+	require.NoError(t, err)
+
+	differences := state.Diff(enState, offchainState)
+
+	require.Len(t, differences, 0)
+}
+
+func EVMStateDiff(t *testing.T) {
+
+	state1 := EVMStateFromReplayGobDir(t, "/var/flow2/evm-state-from-gobs-218215348/", uint64(218215348))
+	// state2 := EVMStateFromReplayGobDir(t, "/var/flow2/evm-state-from-gobs-218215348/", uint64(218215348))
+	state2 := EVMStateFromCheckpointExtract(t, "/var/flow2/evm-state-from-checkpoint-218215348/")
+
+	differences := state.Diff(state1, state2)
+
+	for i, diff := range differences {
+		fmt.Printf("Difference %d: %v\n", i, diff)
+	}
+
+	require.Len(t, differences, 0)
+}
+
+func EVMStateFromCheckpointExtract(t *testing.T, dir string) *state.EVMState {
+	enState, err := state.ImportEVMStateFromGob("/var/flow2/evm-state-from-gobs-218215348/")
+	require.NoError(t, err)
+	return enState
+}
+
+func EVMStateFromReplayGobDir(t *testing.T, gobDir string, flowHeight uint64) *state.EVMState {
+	valueFileName, allocatorFileName := evmStateGobFileNamesByEndHeight(gobDir, flowHeight)
+	chainID := flow.Testnet
+
+	allocatorGobs, err := testutils.DeserializeAllocator(allocatorFileName)
+	require.NoError(t, err)
+
+	storageRoot := evm.StorageAccountAddress(chainID)
+	valuesGob, err := testutils.DeserializeState(valueFileName)
+	require.NoError(t, err)
+
+	store := testutils.GetSimpleValueStorePopulated(valuesGob, allocatorGobs)
+
+	bv, err := state.NewBaseView(store, storageRoot)
+	require.NoError(t, err)
+
+	evmState, err := state.Extract(storageRoot, bv)
+	require.NoError(t, err)
+	return evmState
+}
+
+func evmStateGobFileNamesByEndHeight(evmStateGobDir string, endHeight uint64) (string, string) {
+	valueFileName := filepath.Join(evmStateGobDir, fmt.Sprintf("values-%d.gob", endHeight))
+	allocatorFileName := filepath.Join(evmStateGobDir, fmt.Sprintf("allocators-%d.gob", endHeight))
+	return valueFileName, allocatorFileName
+}