Merge pull request #8091 from onflow/tim/7851-bootstrap-clustering-test

Collection cluster bootstrapping+voting test

Author: tim-barry

cmd/bootstrap/cmd/clustering.go

@@ -120,7 +120,7 @@ func clusterAssignment(cmd *cobra.Command, args []string) {
 	}
 
 	log.Info().Msg("computing collection node clusters")
-	assignments, clusters, err := common.ConstructClusterAssignment(log, partnerList, internalList, int(flagCollectionClusters), clusteringPrg)
+	assignments, clusters, canConstructQCs, err := common.ConstructClusterAssignment(log, partnerList, internalList, int(flagCollectionClusters), clusteringPrg)
 	if err != nil {
 		log.Fatal().Err(err).Msg("unable to generate cluster assignment")
 	}
@@ -145,6 +145,12 @@ func clusterAssignment(cmd *cobra.Command, args []string) {
 		model.FilterByRole(internalNodes, flow.RoleCollection),
 	)
 	log.Info().Msg("")
+
+	if canConstructQCs {
+		log.Info().Msg("enough votes for collection clusters are present - bootstrapping can continue with root block creation")
+	} else {
+		log.Info().Msg("not enough internal votes to generate cluster QCs, need partner votes before root block creation")
+	}
 }
 
 // constructClusterRootVotes generates and writes vote files for internal collector nodes with private keys available.

cmd/bootstrap/cmd/constraints.go

@@ -26,8 +26,7 @@ func ensureUniformNodeWeightsPerRole(allNodes flow.IdentityList) {
 	}
 }
 
-// Checks constraints about the number of partner and internal nodes.
-//   - Internal nodes must comprise >2/3 of each collector cluster.
+// Checks constraints about the weights of partner and internal nodes.
 //   - for all roles R:
 //     all node with role R must have the same weight
 func checkConstraints(partnerNodes, internalNodes []model.NodeInfo) {

cmd/bootstrap/cmd/finalize_test.go

@@ -122,6 +122,7 @@ func TestClusterAssignment(t *testing.T) {
 	// Happy path (limit set-up, can't have one less internal node)
 	partnersLen := 7
 	internalLen := 22
+	// clusters are assigned with ratios (partner:internal) [2:5, 2:5, 1:4, 1:4, 1:4]
 	partners := unittest.NodeInfosFixture(partnersLen, unittest.WithRole(flow.RoleCollection))
 	internals := unittest.NodeInfosFixture(internalLen, unittest.WithRole(flow.RoleCollection))
 
@@ -134,15 +135,16 @@ func TestClusterAssignment(t *testing.T) {
 
 	log := zerolog.Nop()
 	// should not error
-	_, clusters, err := common.ConstructClusterAssignment(log, model.ToIdentityList(partners), model.ToIdentityList(internals), int(flagCollectionClusters), prng)
+	_, _, canConstructQCs, err := common.ConstructClusterAssignment(log, model.ToIdentityList(partners), model.ToIdentityList(internals), int(flagCollectionClusters), prng)
 	require.NoError(t, err)
-	require.True(t, checkClusterConstraint(clusters, partners, internals))
+	require.True(t, canConstructQCs)
 
 	// unhappy Path
-	internals = internals[:21] // reduce one internal node
-	// should error
-	_, _, err = common.ConstructClusterAssignment(log, model.ToIdentityList(partners), model.ToIdentityList(internals), int(flagCollectionClusters), prng)
-	require.Error(t, err)
+	internals = internals[:len(internals)-1] // reduce one internal node
+	// should no longer be able to construct QCs using only votes from internal nodes
+	_, _, canConstructQCs, err = common.ConstructClusterAssignment(log, model.ToIdentityList(partners), model.ToIdentityList(internals), int(flagCollectionClusters), prng)
+	require.NoError(t, err)
+	require.False(t, canConstructQCs)
 	// revert the flag value
 	flagCollectionClusters = tmp
 }
@@ -190,28 +192,6 @@ func TestEpochTimingConfig(t *testing.T) {
 	})
 }
 
-// Check about the number of internal/partner nodes in each cluster. The identites
-// in each cluster do not matter for this check.
-func checkClusterConstraint(clusters flow.ClusterList, partnersInfo []model.NodeInfo, internalsInfo []model.NodeInfo) bool {
-	partners := model.ToIdentityList(partnersInfo)
-	internals := model.ToIdentityList(internalsInfo)
-	for _, cluster := range clusters {
-		var clusterPartnerCount, clusterInternalCount int
-		for _, node := range cluster {
-			if _, exists := partners.ByNodeID(node.NodeID); exists {
-				clusterPartnerCount++
-			}
-			if _, exists := internals.ByNodeID(node.NodeID); exists {
-				clusterInternalCount++
-			}
-		}
-		if clusterInternalCount <= clusterPartnerCount*2 {
-			return false
-		}
-	}
-	return true
-}
-
 func TestMergeNodeInfos(t *testing.T) {
 	partnersLen := 7
 	internalLen := 22

cmd/bootstrap/cmd/rootblock.go

@@ -228,10 +228,6 @@ func rootBlock(cmd *cobra.Command, args []string) {
 		log.Fatal().Err(err).Msgf("failed to merge node infos")
 	}
 
-	log.Info().Msg("running DKG for consensus nodes")
-	randomBeaconData, dkgIndexMap := runBeaconKG(model.FilterByRole(stakingNodes, flow.RoleConsensus))
-	log.Info().Msg("")
-
 	// create flow.IdentityList representation of the participant set
 	participants := model.ToIdentityList(stakingNodes).Sort(flow.Canonical[flow.Identity])
 
@@ -253,6 +249,10 @@ func rootBlock(cmd *cobra.Command, args []string) {
 	clusterQCs := run.ConstructClusterRootQCsFromVotes(log, clusters, internalNodes, clusterBlocks, votes)
 	log.Info().Msg("")
 
+	log.Info().Msg("running DKG for consensus nodes")
+	randomBeaconData, dkgIndexMap := runBeaconKG(model.FilterByRole(stakingNodes, flow.RoleConsensus))
+	log.Info().Msg("")
+
 	log.Info().Msg("constructing root header")
 	headerBody, err := constructRootHeaderBody(flagRootChain, flagRootParent, flagRootHeight, flagRootView, flagRootTimestamp)
 	if err != nil {

cmd/bootstrap/cmd/rootblock_test.go

@@ -28,18 +28,18 @@ const rootBlockHappyPathLogs = "collecting partner network and staking keys" +
 	`removed 0 internal partner nodes` +
 	`checking constraints on consensus nodes` +
 	`assembling network and staking keys` +
-	`running DKG for consensus nodes` +
-	`read \d+ node infos for DKG` +
-	`will run DKG` +
-	`finished running DKG` +
-	`.+/random-beacon.priv.json` +
-	`wrote file \S+/root-dkg-data.priv.json` +
 	`reading votes for collection node cluster root blocks` +
 	`read vote .+` +
 	`constructing root blocks for collection node clusters` +
 	`constructing root QCs for collection node clusters` +
 	`producing QC for cluster .*` +
 	`producing QC for cluster .*` +
+	`running DKG for consensus nodes` +
+	`read \d+ node infos for DKG` +
+	`will run DKG` +
+	`finished running DKG` +
+	`.+/random-beacon.priv.json` +
+	`wrote file \S+/root-dkg-data.priv.json` +
 	`constructing root header` +
 	`constructing intermediary bootstrapping data` +
 	`wrote file \S+/intermediary-bootstrapping-data.json` +

cmd/bootstrap/run/epochs.go

@@ -181,7 +181,7 @@ func GenerateRecoverTxArgsWithDKG(
 		return nil, fmt.Errorf("could not initialize PRNG: %w", err)
 	}
 	log.Info().Msgf("partitioning %d partners + %d internal nodes into %d collector clusters", len(partnerCollectors), len(internalCollectors), collectionClusters)
-	assignments, clusters, err := common.ConstructClusterAssignment(log, partnerCollectors, internalCollectors, collectionClusters, rng)
+	assignments, clusters, _, err := common.ConstructClusterAssignment(log, partnerCollectors, internalCollectors, collectionClusters, rng)
 	if err != nil {
 		return nil, fmt.Errorf("unable to generate cluster assignment: %w", err)
 	}

cmd/bootstrap/test/.gitignore

@@ -0,0 +1,3 @@
+*
+!permissionless-cluster-qc-voting-transit-test.sh
+!.gitignore

cmd/bootstrap/test/permissionless-cluster-qc-voting-transit-test.sh

@@ -0,0 +1,187 @@
+#!/usr/bin/env bash
+
+# This file implements a test for bootstrapping a network while having <2/3rds of private keys available for Collector nodes.
+# In particular, it tests decentralized voting by Collection nodes on root cluster blocks as part of the bootstrapping process.
+# The test can be run using either local or GCP bucket vote transport. To test GCP bucket transport, set the `bucket` and `token` variables below.
+# To run this test, you must have `jq` and `gsutil` installed.
+
+bucket=
+token=
+
+bootstrapcmd="go run .."
+transitcmd="go run ../transit"
+
+clusteringpath="public-root-information/root-clustering.json"
+clustervotespath="public-root-information/root-block-votes"
+# partner dir must end with `public-root-information`, otherwise partner nodeinfo will not be read from it
+partner_dir="./public-root-information"
+# keygen dir must not exist yet, or be empty
+keygen_dir="./keygen"
+
+clusterCount=2
+
+# exit early if anything fails
+set -e
+# avoid overwriting existing files or using data from a previous run
+if [ "$(ls | wc -l)" -gt 1 ]
+then
+  echo "Found files in $(pwd), please clean up after previous runs:"
+  echo "rm -rf permissionless public-root-information private-root-information execution-state $keygen_dir $partner_dir node-config.json partner-weights.json"
+  exit 1
+fi
+
+$bootstrapcmd genconfig --address-format "%s%d.example.com:3569" \
+    --access 2 --collection 7 --consensus 3 --execution 2 --verification 1 --weight 100 \
+    -o ./ --config ./node-config.json
+$bootstrapcmd keygen --config ./node-config.json -o "$keygen_dir"
+
+
+echo "simulating permissionless nodes"
+# mark >33% of collectors permissionless / non-internal
+permissionless_collectors=$(jq -r 'map(select(.["Role"]=="collection")) | .[:(length/3|floor)+1] | .[] | .["NodeID"]' \
+    -- "$keygen_dir/public-root-information/node-internal-infos.pub.json")
+
+# generate partner-weights file
+jq 'map({(.["NodeID"]):.["Weight"]}) | add' \
+    -- "$keygen_dir/public-root-information/node-internal-infos.pub.json" > ./partner-weights.json
+
+# generate partner-node-infos (for non-internal nodes only)
+mkdir -p "$partner_dir"
+for node in $permissionless_collectors
+do
+  jq --arg jq_node_id "$node" '.[] | select(.["NodeID"]==$jq_node_id)' \
+      -- "$keygen_dir/public-root-information/node-internal-infos.pub.json" \
+      > "$partner_dir/node-info.pub.$node.json"
+done
+
+# create a directory for each permissionless node to store its private information
+for node in $permissionless_collectors
+do
+  mkdir -p "./permissionless/$node/private-root-information"
+  mkdir -p "./permissionless/$node/public-root-information"
+  echo "$node" >> "./permissionless/$node/public-root-information/node-id"
+  mv "$keygen_dir/private-root-information/private-node-info_$node" "./permissionless/$node/private-root-information/"
+done
+
+
+$bootstrapcmd cluster-assignment \
+    --epoch-counter 0 \
+    --collection-clusters $clusterCount \
+    --clustering-random-seed 00000000000000000000000000000000000000000000000000000000deadbeef \
+    --config ./node-config.json \
+    -o ./ \
+    --partner-dir "$partner_dir" \
+    --partner-weights ./partner-weights.json \
+    --internal-priv-dir "$keygen_dir/private-root-information"
+
+
+#confirm that the bootstrapping process cannot continue yet (not enough votes for cluster QCs)
+echo "Expecting FTL (not enough votes for Cluster QCs)..."
+$bootstrapcmd rootblock \
+    --root-chain bench \
+    --root-height 0 \
+    --root-parent 0000000000000000000000000000000000000000000000000000000000000000 \
+    --root-view 0 \
+    --epoch-counter 0 \
+    --epoch-length 30000 \
+    --epoch-staking-phase-length 20000 \
+    --epoch-dkg-phase-length 2000 \
+    --random-seed 00000000000000000000000000000000000000000000000000000000deadbeef \
+    --collection-clusters $clusterCount \
+    --use-default-epoch-timing \
+    --kvstore-finalization-safety-threshold=1000 \
+    --kvstore-epoch-extension-view-count=2000 \
+    --config ./node-config.json \
+    -o ./ \
+    --partner-dir "$partner_dir" \
+    --partner-weights ./partner-weights.json \
+    --internal-priv-dir "$keygen_dir/private-root-information" \
+    --intermediary-clustering-data "./$clusteringpath" \
+    --cluster-votes-dir "./$clustervotespath" \
+    | grep "not enough votes to create qc"
+
+echo "Collecting votes for Cluster QCs..."
+# permissionless collectors retrieve the clustering data
+if [ -n "$bucket" ]
+then
+  # upload to cloud bucket; collectors pull using transit script
+  echo "uploading to cloud bucket..."
+  gsutil cp "./$clusteringpath" "gs://$bucket/$token/$clusteringpath"
+  for node in $permissionless_collectors
+  do
+    $transitcmd pull-clustering -g "$bucket" -t "$token" -b "./permissionless/$node"
+  done
+else
+  # copy locally
+  for node in $permissionless_collectors
+  do
+    cp "./$clusteringpath" "./permissionless/$node/$clusteringpath"
+  done
+fi
+
+# cluster voting
+for node in $permissionless_collectors
+do
+  $transitcmd generate-cluster-block-vote -b "./permissionless/$node"
+done
+
+# collectors push votes, and bootstrapping machine retrieves them
+if [ -n "$bucket" ]
+then
+  # collectors push using transit script
+  for node in $permissionless_collectors
+  do
+    $transitcmd push-cluster-block-vote -g "$bucket" -t "$token" -b "./permissionless/$node"
+  done
+  gsutil cp "gs://$bucket/$token/root-cluster-block-vote.*" "./$clustervotespath/"
+else
+  # copy locally
+  for node in $permissionless_collectors
+  do
+    cp "./permissionless/$node/private-root-information/private-node-info_$node/root-cluster-block-vote.json" "./$clustervotespath/root-cluster-block-vote.$node.json"
+  done
+fi
+
+
+# root block creation should succeed now that we have enough votes
+$bootstrapcmd rootblock \
+    --root-chain bench \
+    --root-height 0 \
+    --root-parent 0000000000000000000000000000000000000000000000000000000000000000 \
+    --root-view 0 \
+    --epoch-counter 0 \
+    --epoch-length 30000 \
+    --epoch-staking-phase-length 20000 \
+    --epoch-dkg-phase-length 2000 \
+    --random-seed 00000000000000000000000000000000000000000000000000000000deadbeef \
+    --collection-clusters $clusterCount \
+    --use-default-epoch-timing \
+    --kvstore-finalization-safety-threshold=1000 \
+    --kvstore-epoch-extension-view-count=2000 \
+    --config ./node-config.json \
+    -o ./ \
+    --partner-dir "$partner_dir" \
+    --partner-weights ./partner-weights.json \
+    --internal-priv-dir "$keygen_dir/private-root-information" \
+    --intermediary-clustering-data "./$clusteringpath" \
+    --cluster-votes-dir "./$clustervotespath"
+
+
+# root block finalization should succeed with enough consensus votes
+$bootstrapcmd finalize \
+    --config ./node-config.json \
+    --partner-dir "./$partner_dir" \
+    --partner-weights ./partner-weights.json \
+    --internal-priv-dir "$keygen_dir/private-root-information" \
+    --dkg-data ./private-root-information/root-dkg-data.priv.json \
+    --root-block ./public-root-information/root-block.json \
+    --intermediary-bootstrapping-data ./public-root-information/intermediary-bootstrapping-data.json \
+    --root-block-votes-dir ./public-root-information/root-block-votes/ \
+    --root-commit 0000000000000000000000000000000000000000000000000000000000000000 \
+    --genesis-token-supply="1000000000.0" \
+    --service-account-public-key-json "{\"PublicKey\":\"R7MTEDdLclRLrj2MI1hcp4ucgRTpR15PCHAWLM5nks6Y3H7+PGkfZTP2di2jbITooWO4DD1yqaBSAVK8iQ6i0A==\",\"SignAlgo\":2,\"HashAlgo\":1,\"SeqNumber\":0,\"Weight\":1000}" \
+    -o ./
+
+# allow output to be inspected if necessary
+echo "To clean up results, run:"
+echo "rm -rf permissionless public-root-information private-root-information execution-state $keygen_dir $partner_dir node-config.json partner-weights.json"

cmd/bootstrap/transit/cmd/generate_cluster_block_vote.go

@@ -34,7 +34,7 @@ func addGenerateClusterVoteCmdFlags() {
 }
 
 func generateClusterVote(c *cobra.Command, args []string) {
-	log.Info().Msg("generating root block vote")
+	log.Info().Msg("generating cluster block vote")
 
 	nodeIDString, err := readNodeID()
 	if err != nil {

cmd/bootstrap/utils/unittest.go

@@ -13,7 +13,7 @@ func RunWithSporkBootstrapDir(t testing.TB, f func(bootDir, partnerDir, partnerW
 	dir := unittest.TempDir(t)
 	defer os.RemoveAll(dir)
 
-	// make sure constraints are satisfied, 2/3's of con and col nodes are internal
+	// make sure constraints are satisfied to create QCs locally: >2/3's of con and col nodes are internal
 	internalNodes := GenerateNodeInfos(3, 6, 2, 1, 1)
 	partnerNodes := GenerateNodeInfos(1, 1, 1, 1, 1)