Merge branch 'master' into sre/jp/remove-unnecessary-env-check

Author: sjonpaulbrown

Makefile

@@ -251,12 +251,12 @@ tools/custom-gcl: tools/structwrite .custom-gcl.yml
 .PHONY: lint
 lint: tidy tools/custom-gcl
 	# revive -config revive.toml -exclude storage/ledger/trie ./...
-	./tools/custom-gcl run -v ./...
+	./tools/custom-gcl run -v $(or $(LINT_PATH),./...)
 
 .PHONY: fix-lint
 fix-lint:
 	# revive -config revive.toml -exclude storage/ledger/trie ./...
-	./tools/custom-gcl run -v --fix ./...
+	./tools/custom-gcl run -v --fix $(or $(LINT_PATH),./...)
 
 # Runs unit tests with different list of packages as passed by CI so they run in parallel
 .PHONY: ci

cmd/bootstrap/transit/cmd/crypto_test.go

@@ -14,14 +14,12 @@ import (
 const nodeID string = "0000000000000000000000000000000000000000000000000000000000000001"
 
 func TestEndToEnd(t *testing.T) {
-
 	// Create a temp directory to work as "bootstrap"
 	bootdir := t.TempDir()
 
 	t.Logf("Created dir %s", bootdir)
 
 	// Create test files
-	//bootcmd.WriteText(filepath.Join(bootdir, bootstrap.PathNodeId), []byte(nodeID)
 	randomBeaconPath := filepath.Join(bootdir, fmt.Sprintf(bootstrap.PathRandomBeaconPriv, nodeID))
 	err := os.MkdirAll(filepath.Dir(randomBeaconPath), 0755)
 	if err != nil {
@@ -46,9 +44,13 @@ func TestEndToEnd(t *testing.T) {
 		t.Fatalf("Error wrapping files: %s", err)
 	}
 
+	unWrappedFilePath := filepath.Join(
+		bootdir,
+		fmt.Sprintf(bootstrap.PathRandomBeaconPriv, nodeID),
+	)
 	// Client:
 	// Unwrap files
-	err = unWrapFile(bootdir, nodeID)
+	err = unWrapFile(bootdir, nodeID, bootdir, unWrappedFilePath)
 	if err != nil {
 		t.Fatalf("Error unwrapping response: %s", err)
 	}

cmd/bootstrap/transit/cmd/flags.go

@@ -11,8 +11,9 @@ var (
 	flagTimeout       time.Duration
 	flagConcurrency   int64
 
-	flagWrapID    string // wrap ID
-	flagVoteFile  string
-	flagNodeID    string
-	flagOutputDir string
+	flagWrapID       string // wrap ID
+	flagVoteFile     string
+	flagVoteFilePath string
+	flagNodeID       string
+	flagOutputDir    string
 )

cmd/bootstrap/transit/cmd/generate_root_block_vote.go

@@ -26,6 +26,11 @@ var generateVoteCmd = &cobra.Command{
 
 func init() {
 	rootCmd.AddCommand(generateVoteCmd)
+	addGenerateVoteCmdFlags()
+}
+
+func addGenerateVoteCmdFlags() {
+	generateVoteCmd.Flags().StringVarP(&flagOutputDir, "outputDir", "o", "", "ouput directory for vote files; if not set defaults to bootstrap directory")
 }
 
 func generateVote(c *cobra.Command, args []string) {
@@ -78,6 +83,12 @@ func generateVote(c *cobra.Command, args []string) {
 	signer := verification.NewCombinedSigner(me, beaconKeyStore)
 
 	path = filepath.Join(flagBootDir, bootstrap.PathRootBlockData)
+
+	// If output directory is specified, use it for the root-block.json
+	if flagOutputDir != "" {
+		path = filepath.Join(flagOutputDir, "root-block.json")
+	}
+
 	data, err = io.ReadFile(path)
 	if err != nil {
 		log.Fatal().Err(err).Msg("could not read root block file")
@@ -96,7 +107,15 @@ func generateVote(c *cobra.Command, args []string) {
 
 	voteFile := fmt.Sprintf(bootstrap.PathNodeRootBlockVote, nodeID)
 
-	if err = io.WriteJSON(filepath.Join(flagBootDir, voteFile), vote); err != nil {
+	// By default, use the bootstrap directory for storing the vote file
+	voteFilePath := filepath.Join(flagBootDir, voteFile)
+
+	// If output directory is specified, use it for the vote file path
+	if flagOutputDir != "" {
+		voteFilePath = filepath.Join(flagOutputDir, "root-block-vote.json")
+	}
+
+	if err = io.WriteJSON(voteFilePath, vote); err != nil {
 		log.Fatal().Err(err).Msg("could not write vote to file")
 	}
 

cmd/bootstrap/transit/cmd/pull.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/onflow/flow-go/cmd/bootstrap/gcs"
 	"github.com/onflow/flow-go/cmd/bootstrap/utils"
+	"github.com/onflow/flow-go/model/bootstrap"
 	model "github.com/onflow/flow-go/model/bootstrap"
 	"github.com/onflow/flow-go/model/flow"
 )
@@ -156,7 +157,8 @@ func pull(cmd *cobra.Command, args []string) {
 
 	// unwrap consensus node role files
 	if role == flow.RoleConsensus {
-		err = unWrapFile(flagBootDir, nodeID)
+		unWrappedFilePath := filepath.Join(flagBootDir, fmt.Sprintf(bootstrap.PathRandomBeaconPriv, nodeID))
+		err = unWrapFile(flagBootDir, nodeID, flagBootDir, unWrappedFilePath)
 		if err != nil {
 			log.Fatal().Err(err).Msg("failed to pull")
 		}

cmd/bootstrap/transit/cmd/pull_root_block.go

@@ -26,6 +26,7 @@ func init() {
 func addPullRootBlockCmdFlags() {
 	pullRootBlockCmd.Flags().StringVarP(&flagToken, "token", "t", "", "token provided by the Flow team to access the Transit server")
 	pullRootBlockCmd.Flags().StringVarP(&flagBucketName, "bucket-name", "g", "flow-genesis-bootstrap", `bucket for pulling root block`)
+	pullRootBlockCmd.Flags().StringVarP(&flagOutputDir, "outputDir", "o", "", "output directory for root block file; if not set defaults to bootstrap directory")
 	_ = pullRootBlockCmd.MarkFlagRequired("token")
 }
 
@@ -51,28 +52,45 @@ func pullRootBlock(c *cobra.Command, args []string) {
 	log.Info().Msg("downloading root block")
 
 	rootBlockFile := filepath.Join(flagToken, bootstrap.PathRootBlockData)
-	fullOutpath := filepath.Join(flagBootDir, bootstrap.PathRootBlockData)
+	fullRootBlockPath := filepath.Join(flagBootDir, bootstrap.PathRootBlockData)
+	if flagOutputDir != "" {
+		fullRootBlockPath = filepath.Join(flagOutputDir, "root-block.json")
+	}
 
-	log.Info().Str("source", rootBlockFile).Str("dest", fullOutpath).Msgf("downloading root block file from transit servers")
-	err = bucket.DownloadFile(ctx, client, fullOutpath, rootBlockFile)
+	log.Info().Str("source", rootBlockFile).Str("dest", fullRootBlockPath).Msgf("downloading root block file from transit servers")
+	err = bucket.DownloadFile(ctx, client, fullRootBlockPath, rootBlockFile)
 	if err != nil {
 		log.Fatal().Err(err).Msgf("could not download google bucket file")
 	}
 
-	objectName := filepath.Join(flagToken, fmt.Sprintf(FilenameRandomBeaconCipher, nodeID))
-	fullOutpath = filepath.Join(flagBootDir, filepath.Base(objectName))
+	log.Info().Msg("successfully downloaded root block ")
 
-	log.Info().Msgf("downloading random beacon key: %s", objectName)
+	objectName := filepath.Join(flagToken, fmt.Sprintf(FilenameRandomBeaconCipher, nodeID))
 
-	err = bucket.DownloadFile(ctx, client, fullOutpath, objectName)
-	if err != nil {
-		log.Fatal().Err(err).Msg("could not download file from google bucket")
+	// By default, use the bootstrap directory for the random beacon download & unwrapping
+	fullRandomBeaconPath := filepath.Join(flagBootDir, filepath.Base(objectName))
+	unWrappedRandomBeaconPath := filepath.Join(
+		flagBootDir,
+		fmt.Sprintf(bootstrap.PathRandomBeaconPriv, nodeID),
+	)
+
+	// If output directory is specified, use it for the random beacon path
+	// this will set the path used to download the random beacon file and unwrap it
+	if flagOutputDir != "" {
+		fullRandomBeaconPath = filepath.Join(flagOutputDir, filepath.Base(objectName))
+		unWrappedRandomBeaconPath = filepath.Join(flagOutputDir, filepath.Base(objectName))
 	}
 
-	err = unWrapFile(flagBootDir, nodeID)
+	log.Info().Msgf("downloading random beacon key: %s", objectName)
+
+	err = bucket.DownloadFile(ctx, client, fullRandomBeaconPath, objectName)
 	if err != nil {
-		log.Fatal().Err(err).Msg("could not unwrap random beacon file")
+		log.Fatal().Err(err).Msg("could not download random beacon key file from google bucket")
+	} else {
+		err = unWrapFile(flagBootDir, nodeID, flagOutputDir, unWrappedRandomBeaconPath)
+		if err != nil {
+			log.Fatal().Err(err).Msg("could not unwrap random beacon file")
+		}
+		log.Info().Msg("successfully downloaded and unwrapped random beacon private key")
 	}
-
-	log.Info().Msg("successfully downloaded root block and random beacon key")
 }

cmd/bootstrap/transit/cmd/push_root_block_vote.go

@@ -28,9 +28,11 @@ func addPushVoteCmdFlags() {
 	defaultVoteFilePath := fmt.Sprintf(bootstrap.PathNodeRootBlockVote, "<node_id>")
 	pushVoteCmd.Flags().StringVarP(&flagToken, "token", "t", "", "token provided by the Flow team to access the Transit server")
 	pushVoteCmd.Flags().StringVarP(&flagVoteFile, "vote-file", "v", "", fmt.Sprintf("path under bootstrap directory of the vote file to upload (default: %s)", defaultVoteFilePath))
+	pushVoteCmd.Flags().StringVarP(&flagVoteFilePath, "vote-file-dir", "d", "", "directory for vote file to upload, ONLY for vote files outside the bootstrap directory")
 	pushVoteCmd.Flags().StringVarP(&flagBucketName, "bucket-name", "g", "flow-genesis-bootstrap", `bucket for pushing root block vote files`)
 
 	_ = pushVoteCmd.MarkFlagRequired("token")
+	pushVoteCmd.MarkFlagsMutuallyExclusive("vote-file", "vote-file-dir")
 }
 
 func pushVote(c *cobra.Command, args []string) {
@@ -45,12 +47,22 @@ func pushVote(c *cobra.Command, args []string) {
 	}
 
 	voteFile := flagVoteFile
+
+	// If --vote-file-dir is not specified, use the bootstrap directory
+	voteFilePath := filepath.Join(flagBootDir, voteFile)
+
+	// if --vote-file is not specified, use default file name within bootstrap directory
 	if voteFile == "" {
 		voteFile = fmt.Sprintf(bootstrap.PathNodeRootBlockVote, nodeID)
+		voteFilePath = filepath.Join(flagBootDir, voteFile)
+	}
+
+	// If vote-file-dir is specified, use it to construct the full path to the vote file (with default file name)
+	if flagVoteFilePath != "" {
+		voteFilePath = filepath.Join(flagVoteFilePath, "root-block-vote.json")
 	}
 
 	destination := filepath.Join(flagToken, fmt.Sprintf(bootstrap.FilenameRootBlockVote, nodeID))
-	source := filepath.Join(flagBootDir, voteFile)
 
 	log.Info().Msg("pushing root block vote")
 
@@ -67,7 +79,7 @@ func pushVote(c *cobra.Command, args []string) {
 	}
 	defer client.Close()
 
-	err = bucket.UploadFile(ctx, client, destination, source)
+	err = bucket.UploadFile(ctx, client, destination, voteFilePath)
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to upload vote file")
 	}

cmd/bootstrap/transit/cmd/utils.go

@@ -75,7 +75,6 @@ func getFileSHA256(file string) (string, error) {
 
 // moveFile moves a file from source to destination where src and dst are full paths including the filename
 func moveFile(src, dst string) error {
-
 	// check if source file exist
 	if !ioutils.FileExists(src) {
 		return fmt.Errorf("file not found: %s", src)
@@ -153,14 +152,12 @@ func moveFile(src, dst string) error {
 	return nil
 }
 
-func unWrapFile(bootDir string, nodeID string) error {
-
+func unWrapFile(bootDir, nodeID, cipherTextDir, plaintextPath string) error {
 	log.Info().Msg("decrypting Random Beacon key")
 
 	pubKeyPath := filepath.Join(bootDir, fmt.Sprintf(FilenameTransitKeyPub, nodeID))
 	privKeyPath := filepath.Join(bootDir, fmt.Sprintf(FilenameTransitKeyPriv, nodeID))
-	ciphertextPath := filepath.Join(bootDir, fmt.Sprintf(FilenameRandomBeaconCipher, nodeID))
-	plaintextPath := filepath.Join(bootDir, fmt.Sprintf(bootstrap.PathRandomBeaconPriv, nodeID))
+	ciphertextPath := filepath.Join(cipherTextDir, fmt.Sprintf(FilenameRandomBeaconCipher, nodeID))
 
 	ciphertext, err := ioutils.ReadFile(ciphertextPath)
 	if err != nil {
@@ -231,7 +228,6 @@ func wrapFile(bootDir string, nodeID string) error {
 
 // generateKeys creates the transit keypair and writes them to disk for later
 func generateKeys(bootDir string, nodeID string) error {
-
 	privPath := filepath.Join(bootDir, fmt.Sprintf(FilenameTransitKeyPriv, nodeID))
 	pubPath := filepath.Join(bootDir, fmt.Sprintf(FilenameTransitKeyPub, nodeID))