Merge branch 'master' into illia-malachyn/6635-tests-for-websocket-controller

Author: illia-malachyn

.github/workflows/builds.yml

@@ -43,6 +43,10 @@ on:
         type: boolean
         description: 'Build amd64 `without_adx` and `without_netgo_without_adx` images, and arm64 images'
         required: false
+      private_build:
+        type: boolean
+        description: 'Build private images'
+        required: false
 
 jobs:
   # matrix_builder generates a matrix that includes the roles selected in the input
@@ -109,16 +113,30 @@ jobs:
         credentials_json: ${{ secrets.GCR_SERVICE_KEY_SECRET }}
     - name: Set up Google Cloud SDK
       uses: google-github-actions/setup-gcloud@v1
-    - name: Authenticate docker with gcloud
+
+    - name: Authenticate Docker with gcloud
       run: |
-        gcloud auth configure-docker
+        if [[ "${{ github.event.inputs.private_build }}" == "true" ]]; then
+          gcloud auth configure-docker us-central1-docker.pkg.dev
+        else
+          gcloud auth configure-docker
+        fi
+
+    - name: Set CONTAINER_REGISTRY
+      id: set-registry
+      run: |
+        if [[ "${{ github.event.inputs.private_build }}" == "true" ]]; then
+          echo "CONTAINER_REGISTRY=${{ vars.PRIVATE_REGISTRY }}" >> $GITHUB_ENV
+        else
+          echo "CONTAINER_REGISTRY=${{ vars.PUBLIC_REGISTRY }}" >> $GITHUB_ENV
+        fi
 
     - name: Build/Push ${{ matrix.role }} amd64 images with adx (default)
       env:
         IMAGE_TAG: ${{ inputs.docker_tag }}
         CADENCE_DEPLOY_KEY: ${{ secrets.CADENCE_DEPLOY_KEY }}
       run: |
-        make docker-build-${{ matrix.role }}-with-adx docker-push-${{ matrix.role }}-with-adx
+        make docker-build-${{ matrix.role }}-with-adx docker-push-${{ matrix.role }}-with-adx CONTAINER_REGISTRY=$CONTAINER_REGISTRY
 
     - name: Build/Push ${{ matrix.role }} amd64 images without netgo and without adx, arm64 images
       if: ${{ inputs.include_alternative_builds }}
@@ -128,7 +146,7 @@ jobs:
       run: |
         make docker-build-${{ matrix.role }}-without-adx docker-push-${{ matrix.role }}-without-adx \
           docker-build-${{ matrix.role }}-without-netgo-without-adx docker-push-${{ matrix.role }}-without-netgo-without-adx \
-          docker-cross-build-${{ matrix.role }}-arm docker-push-${{ matrix.role }}-arm
+          docker-cross-build-${{ matrix.role }}-arm docker-push-${{ matrix.role }}-arm CONTAINER_REGISTRY=$CONTAINER_REGISTRY
 
 
         

Makefile

@@ -860,6 +860,7 @@ docker-all-tools: tool-util tool-remove-execution-fork
 PHONY: docker-build-util
 docker-build-util:
 	docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/util --build-arg GOARCH=$(GOARCH) --build-arg VERSION=$(IMAGE_TAG) --build-arg CGO_FLAG=$(DISABLE_ADX) --target production \
+		--secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \
 		-t "$(CONTAINER_REGISTRY)/util:latest"  \
 		-t "$(CONTAINER_REGISTRY)/util:$(IMAGE_TAG)" .
 

cmd/bootstrap/utils/md5.go

@@ -2,7 +2,8 @@ package utils
 
 // The google storage API only provides md5 and crc32 hence overriding the linter flag for md5
 import (
-	"crypto/md5" //nolint:gosec
+	// #nosec
+	"crypto/md5"
 	"io"
 	"os"
 )

cmd/util/cmd/export-evm-state/cmd.go

@@ -3,13 +3,17 @@ package evm_exporter
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 
+	"github.com/onflow/atree"
+
 	"github.com/onflow/flow-go/cmd/util/ledger/util"
 	"github.com/onflow/flow-go/fvm/evm"
 	"github.com/onflow/flow-go/fvm/evm/emulator/state"
+	"github.com/onflow/flow-go/fvm/evm/testutils"
 	"github.com/onflow/flow-go/ledger"
 	"github.com/onflow/flow-go/ledger/common/convert"
 	"github.com/onflow/flow-go/model/flow"
@@ -20,6 +24,8 @@ var (
 	flagExecutionStateDir string
 	flagOutputDir         string
 	flagStateCommitment   string
+	flagEVMStateGobDir    string
+	flagEVMStateGobHeight uint64
 )
 
 var Cmd = &cobra.Command{
@@ -34,21 +40,33 @@ func init() {
 
 	Cmd.Flags().StringVar(&flagExecutionStateDir, "execution-state-dir", "",
 		"Execution Node state dir (where WAL logs are written")
-	_ = Cmd.MarkFlagRequired("execution-state-dir")
 
 	Cmd.Flags().StringVar(&flagOutputDir, "output-dir", "",
 		"Directory to write new Execution State to")
 	_ = Cmd.MarkFlagRequired("output-dir")
 
 	Cmd.Flags().StringVar(&flagStateCommitment, "state-commitment", "",
 		"State commitment (hex-encoded, 64 characters)")
+
+	Cmd.Flags().StringVar(&flagEVMStateGobDir, "evm_state_gob_dir", "/var/flow/data/evm_state_gob",
+		"directory that stores the evm state gob files as checkpoint")
+
+	Cmd.Flags().Uint64Var(&flagEVMStateGobHeight, "evm_state_gob_height", 0,
+		"the flow height of the evm state gob files")
 }
 
 func run(*cobra.Command, []string) {
 	log.Info().Msg("start exporting evm state")
-	err := ExportEVMState(flagChain, flagExecutionStateDir, flagStateCommitment, flagOutputDir)
-	if err != nil {
-		log.Fatal().Err(err).Msg("cannot get export evm state")
+	if flagExecutionStateDir != "" {
+		err := ExportEVMState(flagChain, flagExecutionStateDir, flagStateCommitment, flagOutputDir)
+		if err != nil {
+			log.Fatal().Err(err).Msg("cannot get export evm state")
+		}
+	} else if flagEVMStateGobDir != "" {
+		err := ExportEVMStateFromGob(flagChain, flagEVMStateGobDir, flagEVMStateGobHeight, flagOutputDir)
+		if err != nil {
+			log.Fatal().Err(err).Msg("cannot get export evm state from gob files")
+		}
 	}
 }
 
@@ -83,7 +101,40 @@ func ExportEVMState(
 
 	payloadsLedger := util.NewPayloadsLedger(filteredPayloads)
 
-	exporter, err := state.NewExporter(payloadsLedger, storageRoot)
+	return ExportEVMStateFromPayloads(payloadsLedger, storageRoot, outputPath)
+}
+
+func ExportEVMStateFromGob(
+	chainName string,
+	evmStateGobDir string,
+	flowHeight uint64,
+	outputPath string) error {
+
+	valueFileName, allocatorFileName := evmStateGobFileNamesByEndHeight(evmStateGobDir, flowHeight)
+	chainID := flow.ChainID(chainName)
+
+	storageRoot := evm.StorageAccountAddress(chainID)
+	valuesGob, err := testutils.DeserializeState(valueFileName)
+	if err != nil {
+		return err
+	}
+
+	allocatorGobs, err := testutils.DeserializeAllocator(allocatorFileName)
+	if err != nil {
+		return err
+	}
+
+	store := testutils.GetSimpleValueStorePopulated(valuesGob, allocatorGobs)
+
+	return ExportEVMStateFromPayloads(store, storageRoot, outputPath)
+}
+
+func ExportEVMStateFromPayloads(
+	ledger atree.Ledger,
+	storageRoot flow.Address,
+	outputPath string,
+) error {
+	exporter, err := state.NewExporter(ledger, storageRoot)
 	if err != nil {
 		return fmt.Errorf("failed to create exporter: %w", err)
 	}
@@ -95,15 +146,15 @@ func ExportEVMState(
 		}
 	}
 
-	fi, err := os.Create(outputPath)
-	if err != nil {
-		return err
-	}
-	defer fi.Close()
-
-	err = exporter.Export(outputPath)
+	err = exporter.ExportGob(outputPath)
 	if err != nil {
 		return fmt.Errorf("failed to export: %w", err)
 	}
 	return nil
 }
+
+func evmStateGobFileNamesByEndHeight(evmStateGobDir string, endHeight uint64) (string, string) {
+	valueFileName := filepath.Join(evmStateGobDir, fmt.Sprintf("values-%d.gob", endHeight))
+	allocatorFileName := filepath.Join(evmStateGobDir, fmt.Sprintf("allocators-%d.gob", endHeight))
+	return valueFileName, allocatorFileName
+}

engine/consensus/dkg/doc.go

@@ -1,54 +1,45 @@
-/*
-
-Package dkg implements engines for the DKG protocol.
-
-ReactorEngine
-
-ReactorEngine implements triggers to control the lifecycle of DKG runs. A new
-DKG protocol is started when an EpochSetup event is sealed and finalized. The
-subsequent phase transitions are triggered when specified views are encountered
-(specifically when the first block of a given view is finalized). In between
-phase transitions the engine regularly queries the DKG smart-contract to read
-broadcast messages.
-
-MessagingEngine
-
-MessagingEngine is a network engine that enables consensus nodes to securely
-exchange private DKG messages. Note that broadcast messages are not exchanged
-through this engine, but rather via the DKG smart-contract.
-
-Architecture
-
-For every new epoch, the ReactorEngine instantiates a new DKGController with a
-new Broker using the provided ControllerFactory. The ControllerFactory ties new
-DKGControllers to the MessagingEngine via a BrokerTunnel which exposes channels
-to relay incoming and outgoing messages (cf. module/dkg).
-
-    EpochSetup/OnView
-          |
-          v
-   +---------------+
-   | ReactorEngine |
-   +---------------+
-          |
-          v
-*~~~~~~~~~~~~~~~~~~~~~* (one/epoch)
-|  +---------------+  |
-|  | Controller    |  |
-|  +---------------+  |
-|         |           |
-|         v           |
-|  +---------------+  |
-|  | Broker        |  |
-|  +---------------+  |
-*~~~~~~~~|~~~~~~~~~\~~*
-       tunnel      smart-contract client
-         |           \
-   +--------------+   +------------------+
-   | Messaging    |   | DKGSmartContract |
-   | Engine       |   |                  |
-   +--------------+   +------------------+
-
-*/
-
+// Package dkg implements engines for the DKG protocol.
+//
+// # Reactor Engine
+//
+// The [ReactorEngine] implements triggers to control the lifecycle of DKG instances.
+// A new DKG instance is started when an EpochSetup service event is sealed.
+// The subsequent phase transitions are triggered when specified views are encountered.
+// Specifically, phase transitions for a view V are triggered when the first block with view ≥V is finalized.
+// Between phase transitions, we periodically query the DKG smart-contract ("whiteboard") to read broadcast messages.
+// Before transitioning the state machine to the next phase, we query the whiteboard w.r.t. the final view
+// of the phase - this ensures all participants eventually observe the same set of messages for each phase.
+//
+// # Messaging Engine
+//
+// The [MessagingEngine] is a network engine that enables consensus nodes to securely exchange
+// private (not broadcast) DKG messages. Broadcast messages are sent via the DKG smart contract.
+//
+// # Architecture
+//
+// In the happy path, one DKG instance runs every epoch. For each DKG instance, the [ReactorEngine]
+// instantiates a new, epoch-scoped module.DKGController and module.DKGBroker using the provided dkg.ControllerFactory.
+// The dkg.ControllerFactory ties new module.DKGController's to the [MessagingEngine] via a dkg.BrokerTunnel,
+// which exposes channels to relay incoming and outgoing messages (see package module/dkg for details).
+//
+//	EpochSetup/EpochCommit/OnView events
+//	              ↓
+//	      ┏━━━━━━━━━━━━━━━━━┓
+//	      ┃   ReactorEngine ┃
+//	      ┗━━━━━━━━━━━━━━━━━┛
+//	              ↓
+//	      ┏━━━━━━━━━━━━━━━━━┓    ╮
+//	      ┃   Controller    ┃    │
+//	      ┗━━━━━━━━━━━━━━━━━┛    │
+//	              ↓              ┝ Epoch-scoped components
+//	      ┏━━━━━━━━━━━━━━━━━┓    │
+//	      ┃       Broker    ┃    │
+//	      ┗━━━━━━━━━━━━━━━━━┛    ╯
+//	          │         │
+//	 BrokerTunnel      DKGContractClient
+//	          ↓         ↓
+//	┏━━━━━━━━━━━━━━┓   ┏━━━━━━━━━━━━━━━━━━┓
+//	┃ Messaging    ┃   ┃ FlowDKG smart    ┃
+//	┃ Engine       ┃   ┃ contract         ┃
+//	┗━━━━━━━━━━━━━━┛   ┗━━━━━━━━━━━━━━━━━━┛
 package dkg

fvm/evm/emulator/state/base.go

@@ -74,13 +74,13 @@ func NewBaseView(ledger atree.Ledger, rootAddress flow.Address) (*BaseView, erro
 	// fetch the account collection, if not exist, create one
 	view.accounts, view.accountSetupOnCommit, err = view.fetchOrCreateCollection(AccountsStorageIDKey)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to fetch or create account collection with key %v: %w", AccountsStorageIDKey, err)
 	}
 
 	// fetch the code collection, if not exist, create one
 	view.codes, view.codeSetupOnCommit, err = view.fetchOrCreateCollection(CodesStorageIDKey)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to fetch or create code collection with key %v: %w", CodesStorageIDKey, err)
 	}
 
 	return view, nil
@@ -485,7 +485,10 @@ func (v *BaseView) fetchOrCreateCollection(path string) (collection *Collection,
 	}
 	if len(collectionID) == 0 {
 		collection, err = v.collectionProvider.NewCollection()
-		return collection, true, err
+		if err != nil {
+			return collection, true, fmt.Errorf("fail to create collection with key %v: %w", path, err)
+		}
+		return collection, true, nil
 	}
 	collection, err = v.collectionProvider.CollectionByID(collectionID)
 	return collection, false, err