Skip to content

Commit 1e4497b

Browse files
tim-barrytim-barry
committed
Add new bootstrapping command to generate cluster assignment
Creation of cluster assignment moved from `rootblock.go` to `clustering.go`.
1 parent 5cecaf4 commit 1e4497b

File tree

2 files changed

+207
-30
lines changed

2 files changed

+207
-30
lines changed

‎cmd/bootstrap/cmd/clustering.go‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
package cmd
2+
3+
import (
4+
"github.com/spf13/cobra"
5+
6+
"github.com/onflow/flow-go/cmd"
7+
"github.com/onflow/flow-go/cmd/util/cmd/common"
8+
model "github.com/onflow/flow-go/model/bootstrap"
9+
"github.com/onflow/flow-go/state/protocol/prg"
10+
)
11+
12+
var (
13+
flagClusteringRandomSeed []byte
14+
)
15+
16+
// clusterAssignmentCmd represents the clusterAssignment command
17+
var clusterAssignmentCmd = &cobra.Command{
18+
Use: "cluster-assignment",
19+
Short: "Generate cluster assignment",
20+
Long: `Generate cluster assignment for collection nodes based on partner and internal node info and weights. Serialize into file with Epoch Counter`,
21+
Run: clusterAssignment,
22+
}
23+
24+
func init() {
25+
rootCmd.AddCommand(clusterAssignmentCmd)
26+
addClusterAssignmentCmdFlags()
27+
}
28+
29+
func addClusterAssignmentCmdFlags() {
30+
// required parameters for network configuration and generation of root node identities
31+
clusterAssignmentCmd.Flags().StringVar(&flagConfig, "config", "",
32+
"path to a JSON file containing multiple node configurations (fields Role, Address, Weight)")
33+
clusterAssignmentCmd.Flags().StringVar(&flagInternalNodePrivInfoDir, "internal-priv-dir", "", "path to directory "+
34+
"containing the output from the `keygen` command for internal nodes")
35+
clusterAssignmentCmd.Flags().StringVar(&flagPartnerNodeInfoDir, "partner-dir", "", "path to directory "+
36+
"containing one JSON file starting with node-info.pub.<NODE_ID>.json for every partner node (fields "+
37+
" in the JSON file: Role, Address, NodeID, NetworkPubKey, StakingPubKey)")
38+
clusterAssignmentCmd.Flags().StringVar(&deprecatedFlagPartnerStakes, "partner-stakes", "", "deprecated: use --partner-weights")
39+
clusterAssignmentCmd.Flags().StringVar(&flagPartnerWeights, "partner-weights", "", "path to a JSON file containing "+
40+
"a map from partner node's NodeID to their stake")
41+
42+
cmd.MarkFlagRequired(clusterAssignmentCmd, "config")
43+
cmd.MarkFlagRequired(clusterAssignmentCmd, "internal-priv-dir")
44+
cmd.MarkFlagRequired(clusterAssignmentCmd, "partner-dir")
45+
cmd.MarkFlagRequired(clusterAssignmentCmd, "partner-weights")
46+
47+
// required parameters for generation of cluster root blocks
48+
clusterAssignmentCmd.Flags().Uint64Var(&flagEpochCounter, "epoch-counter", 0, "epoch counter for the epoch beginning with the root block")
49+
cmd.MarkFlagRequired(clusterAssignmentCmd, "epoch-counter")
50+
51+
clusterAssignmentCmd.Flags().BytesHexVar(&flagClusteringRandomSeed, "clustering-random-seed", nil, "random seed to generate the clustering assignment")
52+
cmd.MarkFlagRequired(clusterAssignmentCmd, "clustering-random-seed")
53+
54+
}
55+
56+
func clusterAssignment(cmd *cobra.Command, args []string) {
57+
// maintain backward compatibility with old flag name
58+
if deprecatedFlagPartnerStakes != "" {
59+
log.Warn().Msg("using deprecated flag --partner-stakes (use --partner-weights instead)")
60+
if flagPartnerWeights == "" {
61+
flagPartnerWeights = deprecatedFlagPartnerStakes
62+
} else {
63+
log.Fatal().Msg("cannot use both --partner-stakes and --partner-weights flags (use only --partner-weights)")
64+
}
65+
}
66+
// Read partner node's information and internal node's information.
67+
// With "internal nodes" we reference nodes, whose private keys we have. In comparison,
68+
// for "partner nodes" we generally do not have their keys. However, we allow some overlap,
69+
// in that we tolerate a configuration where information about an "internal node" is also
70+
// duplicated in the list of "partner nodes".
71+
log.Info().Msg("collecting partner network and staking keys")
72+
rawPartnerNodes, err := common.ReadFullPartnerNodeInfos(log, flagPartnerWeights, flagPartnerNodeInfoDir)
73+
if err != nil {
74+
log.Fatal().Err(err).Msg("failed to read full partner node infos")
75+
}
76+
log.Info().Msg("")
77+
78+
log.Info().Msg("generating internal private networking and staking keys")
79+
internalNodes, err := common.ReadFullInternalNodeInfos(log, flagInternalNodePrivInfoDir, flagConfig)
80+
if err != nil {
81+
log.Fatal().Err(err).Msg("failed to read full internal node infos")
82+
}
83+
log.Info().Msg("")
84+
85+
// we now convert to the strict meaning of: "internal nodes" vs "partner nodes"
86+
// • "internal nodes" we have they private keys for
87+
// • "partner nodes" we don't have the keys for
88+
// • both sets are disjoint (no common nodes)
89+
log.Info().Msg("remove internal partner nodes")
90+
partnerNodes := common.FilterInternalPartners(rawPartnerNodes, internalNodes)
91+
log.Info().Msgf("removed %d internal partner nodes", len(rawPartnerNodes)-len(partnerNodes))
92+
93+
log.Info().Msg("checking constraints on consensus nodes")
94+
checkConstraints(partnerNodes, internalNodes)
95+
log.Info().Msg("")
96+
97+
log.Info().Msg("assembling network and staking keys")
98+
stakingNodes, err := mergeNodeInfos(internalNodes, partnerNodes)
99+
if err != nil {
100+
log.Fatal().Err(err).Msgf("failed to merge node infos")
101+
}
102+
publicInfo, err := model.ToPublicNodeInfoList(stakingNodes)
103+
if err != nil {
104+
log.Fatal().Msg("failed to read public node info")
105+
}
106+
err = common.WriteJSON(model.PathNodeInfosPub, flagOutdir, publicInfo)
107+
if err != nil {
108+
log.Fatal().Err(err).Msg("failed to write json")
109+
}
110+
log.Info().Msgf("wrote file %s/%s", flagOutdir, model.PathNodeInfosPub)
111+
log.Info().Msg("")
112+
113+
// Convert to IdentityList
114+
partnerList := model.ToIdentityList(partnerNodes)
115+
internalList := model.ToIdentityList(internalNodes)
116+
117+
clusteringPrg, err := prg.New(flagClusteringRandomSeed, prg.BootstrapClusterAssignment, nil)
118+
if err != nil {
119+
log.Fatal().Err(err).Msg("failed to initialize pseudorandom generator")
120+
}
121+
122+
log.Info().Msg("computing collection node clusters")
123+
assignments, clusters, err := common.ConstructClusterAssignment(log, partnerList, internalList, int(flagCollectionClusters), clusteringPrg)
124+
if err != nil {
125+
log.Fatal().Err(err).Msg("unable to generate cluster assignment")
126+
}
127+
log.Info().Msg("")
128+
129+
// Output assignment with epoch counter
130+
output := IntermediaryClusteringData{
131+
EpochCounter: flagEpochCounter,
132+
Assignments: assignments,
133+
Clusters: clusters,
134+
}
135+
err = common.WriteJSON(model.PathIntermediaryBootstrappingData, flagOutdir, output)
136+
if err != nil {
137+
log.Fatal().Err(err).Msg("failed to write json")
138+
}
139+
}

‎cmd/bootstrap/cmd/rootblock.go‎

Lines changed: 68 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,17 @@ import (
55
"encoding/hex"
66
"fmt"
77
"strconv"
8+
"strings"
89
"time"
910

1011
"github.com/onflow/cadence"
1112
"github.com/spf13/cobra"
1213

1314
"github.com/onflow/flow-go/cmd"
1415
"github.com/onflow/flow-go/cmd/bootstrap/run"
16+
"github.com/onflow/flow-go/cmd/bootstrap/utils"
1517
"github.com/onflow/flow-go/cmd/util/cmd/common"
18+
hotstuff "github.com/onflow/flow-go/consensus/hotstuff/model"
1619
model "github.com/onflow/flow-go/model/bootstrap"
1720
"github.com/onflow/flow-go/model/dkg"
1821
"github.com/onflow/flow-go/model/encodable"
@@ -43,11 +46,15 @@ var (
4346
flagNumViewsInEpoch uint64
4447
flagNumViewsInStakingAuction uint64
4548
flagNumViewsInDKGPhase uint64
49+
flagEpochRandomSeed []byte
4650
// Epoch target end time config
4751
flagUseDefaultEpochTargetEndTime bool
4852
flagEpochTimingRefCounter uint64
4953
flagEpochTimingRefTimestamp uint64
5054
flagEpochTimingDuration uint64
55+
56+
flagIntermediaryClusteringDataPath string
57+
flagRootClusterBlockVotesDir string
5158
)
5259

5360
// rootBlockCmd represents the rootBlock command
@@ -106,11 +113,13 @@ func addRootBlockCmdFlags() {
106113
rootBlockCmd.Flags().Uint64Var(&flagEpochExtensionViewCount, "kvstore-epoch-extension-view-count", 0, "length of epoch extension in views, default is 100_000 which is approximately 1 day")
107114
rootBlockCmd.Flags().StringVar(&flagKVStoreVersion, "kvstore-version", "default",
108115
"protocol state KVStore version to initialize ('default' or an integer equal to a supported protocol version: '0', '1', '2', ...)")
116+
rootBlockCmd.Flags().BytesHexVar(&flagEpochRandomSeed, "random-seed", nil, "random seed")
109117

110118
cmd.MarkFlagRequired(rootBlockCmd, "root-chain")
111119
cmd.MarkFlagRequired(rootBlockCmd, "root-parent")
112120
cmd.MarkFlagRequired(rootBlockCmd, "root-height")
113121
cmd.MarkFlagRequired(rootBlockCmd, "root-view")
122+
cmd.MarkFlagRequired(rootBlockCmd, "random-seed")
114123

115124
// Epoch timing config - these values must be set identically to `EpochTimingConfig` in the FlowEpoch smart contract.
116125
// See https://github.com/onflow/flow-core-contracts/blob/240579784e9bb8d97d91d0e3213614e25562c078/contracts/epochs/FlowEpoch.cdc#L259-L266
@@ -224,16 +233,6 @@ func rootBlock(cmd *cobra.Command, args []string) {
224233
if err != nil {
225234
log.Fatal().Err(err).Msgf("failed to merge node infos")
226235
}
227-
publicInfo, err := model.ToPublicNodeInfoList(stakingNodes)
228-
if err != nil {
229-
log.Fatal().Msg("failed to read public node info")
230-
}
231-
err = common.WriteJSON(model.PathNodeInfosPub, flagOutdir, publicInfo)
232-
if err != nil {
233-
log.Fatal().Err(err).Msg("failed to write json")
234-
}
235-
log.Info().Msgf("wrote file %s/%s", flagOutdir, model.PathNodeInfosPub)
236-
log.Info().Msg("")
237236

238237
log.Info().Msg("running DKG for consensus nodes")
239238
randomBeaconData, dkgIndexMap := runBeaconKG(model.FilterByRole(stakingNodes, flow.RoleConsensus))
@@ -242,30 +241,22 @@ func rootBlock(cmd *cobra.Command, args []string) {
242241
// create flow.IdentityList representation of the participant set
243242
participants := model.ToIdentityList(stakingNodes).Sort(flow.Canonical[flow.Identity])
244243

245-
// use system randomness to create cluster assignment
246-
// TODO(7848): use randomness provided from the command line
247-
var randomSeed [32]byte
248-
_, err = rand.Read(randomSeed[:])
249-
if err != nil {
250-
log.Fatal().Err(err).Msg("unable to generate random seed")
251-
}
252-
clusteringPrg, err := prg.New(randomSeed[:], prg.BootstrapClusterAssignment, nil)
253-
if err != nil {
254-
log.Fatal().Err(err).Msg("unable to initialize pseudorandom generator")
255-
}
256-
log.Info().Msg("computing collection node clusters")
257-
assignments, clusters, err := common.ConstructClusterAssignment(log, model.ToIdentityList(partnerNodes), model.ToIdentityList(internalNodes), int(flagCollectionClusters), clusteringPrg)
258-
if err != nil {
259-
log.Fatal().Err(err).Msg("unable to generate cluster assignment")
244+
// read the previously created cluster assignment (see cmd/bootstrap/cmd/clustering.go)
245+
clusteringData := readIntermediaryClusteringData()
246+
if flagEpochCounter != clusteringData.EpochCounter {
247+
log.Fatal().Msgf("Epoch counter does not match the one used to generate collector clusters")
260248
}
249+
clusters := clusteringData.Clusters
250+
log.Info().Msg("reading votes for collection node cluster root blocks")
251+
votes := readClusterBlockVotes(clusteringData.Assignments)
261252
log.Info().Msg("")
262253

263254
log.Info().Msg("constructing root blocks for collection node clusters")
264-
clusterBlocks := run.GenerateRootClusterBlocks(flagEpochCounter, clusters)
255+
clusterBlocks := run.GenerateRootClusterBlocks(clusteringData.EpochCounter, clusters)
265256
log.Info().Msg("")
266257

267258
log.Info().Msg("constructing root QCs for collection node clusters")
268-
clusterQCs := run.ConstructRootQCsForClusters(log, clusters, internalNodes, clusterBlocks)
259+
clusterQCs := run.ConstructClusterRootQCsFromVotes(log, clusters, internalNodes, clusterBlocks, votes)
269260
log.Info().Msg("")
270261

271262
log.Info().Msg("constructing root header")
@@ -275,13 +266,13 @@ func rootBlock(cmd *cobra.Command, args []string) {
275266
}
276267
log.Info().Msg("")
277268

278-
// use the same randomness for the RandomSource of the new epoch
279-
randomSourcePrg, err := prg.New(randomSeed[:], prg.BootstrapEpochRandomSource, nil)
269+
// use provided randomness for the RandomSource of the new epoch
270+
randomSourcePrg, err := prg.New(flagEpochRandomSeed, prg.BootstrapEpochRandomSource, nil)
280271
if err != nil {
281272
log.Fatal().Err(err).Msg("failed to initialize pseudorandom generator")
282273
}
283274
log.Info().Msg("constructing intermediary bootstrapping data")
284-
epochSetup, epochCommit, err := constructRootEpochEvents(headerBody.View, participants, assignments, clusterQCs, randomBeaconData, dkgIndexMap, randomSourcePrg)
275+
epochSetup, epochCommit, err := constructRootEpochEvents(headerBody.View, participants, clusteringData.Assignments, clusterQCs, randomBeaconData, dkgIndexMap, randomSourcePrg)
285276
if err != nil {
286277
log.Fatal().Err(err).Msg("failed to construct root epoch events")
287278
}
@@ -388,6 +379,53 @@ func validateEpochConfig() error {
388379
return nil
389380
}
390381

382+
// readIntermediaryBootstrappingData reads intermediary clustering data file from disk.
383+
// This file needs to be prepared with the clustering bootstrap command
384+
func readIntermediaryClusteringData() *IntermediaryClusteringData {
385+
intermediaryData, err := utils.ReadData[IntermediaryClusteringData](flagIntermediaryClusteringDataPath)
386+
if err != nil {
387+
log.Fatal().Err(err).Msg("could not read clustering data, was the clustering command run?")
388+
}
389+
return intermediaryData
390+
}
391+
392+
// readClusterBlockVotes reads votes for root cluster blocks.
393+
// It sorts the votes into the appropriate clusters according to the given assignment list.
394+
func readClusterBlockVotes(al flow.AssignmentList) [][]*hotstuff.Vote {
395+
votes := make([][]*hotstuff.Vote, len(al))
396+
files, err := common.FilesInDir(flagRootClusterBlockVotesDir)
397+
if err != nil {
398+
log.Fatal().Err(err).Msg("could not read cluster block votes")
399+
}
400+
for _, f := range files {
401+
// skip files that do not include node-infos
402+
if !strings.Contains(f, model.FilenameClusterBlockVotePrefix) {
403+
continue
404+
}
405+
406+
// read file and append to partners
407+
var vote hotstuff.Vote
408+
err = common.ReadJSON(f, &vote)
409+
if err != nil {
410+
log.Fatal().Err(err).Msg("failed to read json")
411+
}
412+
log.Info().Msgf("read vote %v for block %v from signerID %v", vote.ID(), vote.BlockID, vote.SignerID)
413+
414+
// put the vote in the correct bucket for its cluster
415+
found := false
416+
for i, cluster := range al {
417+
if cluster.Contains(vote.SignerID) {
418+
votes[i] = append(votes[i], &vote)
419+
found = true
420+
}
421+
}
422+
if !found {
423+
log.Warn().Msgf("ignoring vote for block %v from signerID %v not part of the assignment", vote.BlockID, vote.SignerID)
424+
}
425+
}
426+
return votes
427+
}
428+
391429
// generateExecutionStateEpochConfig generates epoch-related configuration used
392430
// to generate an empty root execution state. This config is generated in the
393431
// `rootblock` alongside the root epoch and root protocol state ID for consistency.

0 commit comments

Comments
 (0)