update clustering bootstrap cmd and tests

Author: tim-barry

cmd/bootstrap/cmd/clustering.go

@@ -1,11 +1,18 @@
 package cmd
 
 import (
+	"fmt"
+	"path/filepath"
+
 	"github.com/spf13/cobra"
 
 	"github.com/onflow/flow-go/cmd"
+	"github.com/onflow/flow-go/cmd/bootstrap/run"
 	"github.com/onflow/flow-go/cmd/util/cmd/common"
+	hotstuff "github.com/onflow/flow-go/consensus/hotstuff/model"
 	model "github.com/onflow/flow-go/model/bootstrap"
+	"github.com/onflow/flow-go/model/flow"
+	cluster2 "github.com/onflow/flow-go/state/cluster"
 	"github.com/onflow/flow-go/state/protocol/prg"
 )
 
@@ -132,8 +139,49 @@ func clusterAssignment(cmd *cobra.Command, args []string) {
 		Assignments:  assignments,
 		Clusters:     clusters,
 	}
-	err = common.WriteJSON(model.PathIntermediaryBootstrappingData, flagOutdir, output)
+	err = common.WriteJSON(model.PathClusteringData, flagOutdir, output)
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to write json")
 	}
+	log.Info().Msgf("wrote file %s/%s", flagOutdir, model.PathClusteringData)
+	log.Info().Msg("")
+
+	log.Info().Msg("constructing and writing cluster block votes for internal nodes")
+	constructClusterRootVotes(
+		output,
+		model.FilterByRole(internalNodes, flow.RoleCollection),
+	)
+	log.Info().Msg("")
+}
+
+// constructClusterRootVotes generates and writes vote files for internal collector nodes with private keys available.
+func constructClusterRootVotes(data IntermediaryClusteringData, internalCollectors []model.NodeInfo) {
+	for i := range data.Clusters {
+		clusterRootBlock, err := cluster2.CanonicalRootBlock(data.EpochCounter, data.Assignments[i])
+		if err != nil {
+			log.Fatal().Err(err).Msg("could not construct cluster root block")
+		}
+		block := hotstuff.GenesisBlockFromFlow(clusterRootBlock.ToHeader())
+		// collate private NodeInfos for internal nodes in this cluster
+		signers := make([]model.NodeInfo, 0)
+		for _, nodeID := range data.Assignments[i] {
+			for _, node := range internalCollectors {
+				if node.NodeID == nodeID {
+					signers = append(signers, node)
+				}
+			}
+		}
+		votes, err := run.CreateClusterRootBlockVotes(signers, block)
+		if err != nil {
+			log.Fatal().Err(err).Msg("could not create cluster root block votes")
+		}
+		for _, vote := range votes {
+			path := filepath.Join(model.DirnameRootBlockVotes, fmt.Sprintf(model.FilenameRootClusterBlockVote, vote.SignerID))
+			err = common.WriteJSON(path, flagOutdir, vote)
+			if err != nil {
+				log.Fatal().Err(err).Msg("failed to write json")
+			}
+			log.Info().Msgf("wrote file %s/%s", flagOutdir, path)
+		}
+	}
 }

cmd/bootstrap/cmd/finalize_test.go

@@ -68,12 +68,19 @@ func TestFinalize_HappyPath(t *testing.T) {
 		flagPartnerWeights = partnerWeights
 		flagInternalNodePrivInfoDir = internalPrivDir
 
+		flagIntermediaryClusteringDataPath = filepath.Join(bootDir, model.PathClusteringData)
+		flagRootClusterBlockVotesDir = filepath.Join(bootDir, model.DirnameRootBlockVotes)
+		flagEpochCounter = epochCounter
+
+		// clusterAssignment will generate the collector clusters
+		// In addition, it also generates votes from internal collector nodes
+		clusterAssignment(clusterAssignmentCmd, nil)
+
 		flagRootChain = chainName
 		flagRootParent = hex.EncodeToString(rootParent[:])
 		flagRootHeight = rootHeight
 		flagRootView = 1_000
 		flagRootCommit = hex.EncodeToString(rootCommit[:])
-		flagEpochCounter = epochCounter
 		flagNumViewsInEpoch = 100_000
 		flagNumViewsInStakingAuction = 50_000
 		flagNumViewsInDKGPhase = 2_000

cmd/bootstrap/cmd/rootblock.go

@@ -114,12 +114,14 @@ func addRootBlockCmdFlags() {
 	rootBlockCmd.Flags().StringVar(&flagKVStoreVersion, "kvstore-version", "default",
 		"protocol state KVStore version to initialize ('default' or an integer equal to a supported protocol version: '0', '1', '2', ...)")
 	rootBlockCmd.Flags().BytesHexVar(&flagEpochRandomSeed, "random-seed", nil, "random seed")
+	rootBlockCmd.Flags().StringVar(&flagIntermediaryClusteringDataPath, "intermediary-clustering-data", "", "path to a JSON file containing intermediary clustering data generated by the clustering command")
 
 	cmd.MarkFlagRequired(rootBlockCmd, "root-chain")
 	cmd.MarkFlagRequired(rootBlockCmd, "root-parent")
 	cmd.MarkFlagRequired(rootBlockCmd, "root-height")
 	cmd.MarkFlagRequired(rootBlockCmd, "root-view")
 	cmd.MarkFlagRequired(rootBlockCmd, "random-seed")
+	cmd.MarkFlagRequired(rootBlockCmd, "intermediary-clustering-data")
 
 	// Epoch timing config - these values must be set identically to `EpochTimingConfig` in the FlowEpoch smart contract.
 	// See https://github.com/onflow/flow-core-contracts/blob/240579784e9bb8d97d91d0e3213614e25562c078/contracts/epochs/FlowEpoch.cdc#L259-L266

cmd/bootstrap/cmd/rootblock_test.go

@@ -28,14 +28,14 @@ const rootBlockHappyPathLogs = "collecting partner network and staking keys" +
 	`removed 0 internal partner nodes` +
 	`checking constraints on consensus nodes` +
 	`assembling network and staking keys` +
-	`wrote file \S+/node-infos.pub.json` +
 	`running DKG for consensus nodes` +
 	`read \d+ node infos for DKG` +
 	`will run DKG` +
 	`finished running DKG` +
 	`.+/random-beacon.priv.json` +
 	`wrote file \S+/root-dkg-data.priv.json` +
-	`computing collection node clusters` +
+	`reading votes for collection node cluster root blocks` +
+	`read vote .+` +
 	`constructing root blocks for collection node clusters` +
 	`constructing root QCs for collection node clusters` +
 	`producing QC for cluster .*` +
@@ -59,11 +59,14 @@ func setupHappyPathFlags(bootDir, partnerDir, partnerWeights, internalPrivDir, c
 	flagPartnerWeights = partnerWeights
 	flagInternalNodePrivInfoDir = internalPrivDir
 
+	flagIntermediaryClusteringDataPath = filepath.Join(bootDir, model.PathClusteringData)
+	flagRootClusterBlockVotesDir = filepath.Join(bootDir, model.DirnameRootBlockVotes)
+	flagEpochCounter = 0
+
 	flagRootParent = hex.EncodeToString(rootParent[:])
 	flagRootChain = "main"
 	flagRootHeight = 12332
 	flagRootView = 1000
-	flagEpochCounter = 0
 	flagNumViewsInEpoch = 100_000
 	flagNumViewsInStakingAuction = 50_000
 	flagNumViewsInDKGPhase = 2_000
@@ -81,6 +84,10 @@ func TestRootBlock_HappyPath(t *testing.T) {
 	utils.RunWithSporkBootstrapDir(t, func(bootDir, partnerDir, partnerWeights, internalPrivDir, configPath string) {
 		setupHappyPathFlags(bootDir, partnerDir, partnerWeights, internalPrivDir, configPath)
 
+		// clusterAssignment will generate the collector clusters
+		// In addition, it also generates votes from internal collector nodes
+		clusterAssignment(clusterAssignmentCmd, nil)
+
 		// KV store values (epoch extension view count and finalization safety threshold) must be explicitly set for mainnet
 		require.NoError(t, rootBlockCmd.Flags().Set("kvstore-finalization-safety-threshold", "1000"))
 		require.NoError(t, rootBlockCmd.Flags().Set("kvstore-epoch-extension-view-count", "100000"))

cmd/bootstrap/run/cluster_qc.go

@@ -25,7 +25,7 @@ func GenerateClusterRootQC(signers []bootstrap.NodeInfo, allCommitteeMembers flo
 	clusterRootBlock := model.GenesisBlockFromFlow(clusterBlock.ToHeader())
 
 	// STEP 1: create votes for cluster root block
-	votes, err := createRootBlockVotes(signers, clusterRootBlock)
+	votes, err := CreateClusterRootBlockVotes(signers, clusterRootBlock)
 	if err != nil {
 		return nil, err
 	}
@@ -94,8 +94,8 @@ func createClusterValidator(committee hotstuff.DynamicCommittee) (hotstuff.Valid
 	return hotstuffValidator, nil
 }
 
-// createRootBlockVotes generates a vote for the rootBlock from each participant
-func createRootBlockVotes(participants []bootstrap.NodeInfo, rootBlock *model.Block) ([]*model.Vote, error) {
+// CreateClusterRootBlockVotes generates a vote for the rootBlock from each participant
+func CreateClusterRootBlockVotes(participants []bootstrap.NodeInfo, rootBlock *model.Block) ([]*model.Vote, error) {
 	votes := make([]*model.Vote, 0, len(participants))
 	for _, participant := range participants {
 		// create the participant's local identity