roachprod: add CLI option to download certs dir

This change adds a fetch-certs command that will download
the pgurl certs from a cluster locally. We need a separate
command for certs as opposed to just using roachprod get
as lib/pq will complain about world-readable files.

One benefactor of this command is testing out new roachtest
operations locally. Many operations require certs and having
to manually remember to make them non world readable each
time is non obvious and cumbersome.

Author: DarrylWong

pkg/cmd/roachprod/cli/commands.go

@@ -2207,3 +2207,31 @@ If the time is not provided, it downloads the latest pprof file across all clust
 		}),
 	}
 }
+
+func (cr *commandRegistry) buildFetchCertsDir() *cobra.Command {
+	return &cobra.Command{
+		Use:   "fetch-certs <cluster> [<dest-dir>]",
+		Short: "downloads the PGUrl certs directory from the cluster",
+		Long: fmt.Sprintf(`
+Downloads the PGUrl certs directory from the cluster. In addition to downloading the
+certs, it also makes sure the files are not world readable so lib/pq doesn't complain.
+If a destination is not provided, the certs will be downloaded to a default %s directory.
+
+--certs-dir: specify the directory to download the certs from
+
+`, install.CockroachNodeCertsDir),
+		Args: cobra.RangeArgs(1, 2),
+		Run: wrap(func(cmd *cobra.Command, args []string) error {
+			cluster := args[0]
+			// If a destination is not provided, download the certs to a default directory
+			// for safety. FetchCertsDir will walk the entire directory and chmod each file
+			// so we want to avoid side effects.
+			dest := fmt.Sprintf("./%s", install.CockroachNodeCertsDir)
+			if len(args) == 2 {
+				dest = args[1]
+			}
+			ctx := context.Background()
+			return roachprod.FetchCertsDir(ctx, config.Logger, cluster, pgurlCertsDir, dest)
+		}),
+	}
+}

pkg/cmd/roachprod/cli/resgistry.go

@@ -70,5 +70,6 @@ func (cr *commandRegistry) register() {
 		cr.buildOpentelemetryStopCmd(),
 		cr.buildFetchLogsCmd(),
 		cr.buildGetLatestPProfCmd(),
+		cr.buildFetchCertsDir(),
 	})
 }

pkg/cmd/roachtest/cluster.go

@@ -13,7 +13,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/fs"
 	"log"
 	"math/rand"
 	"net"
@@ -2203,6 +2202,7 @@ func (c *clusterImpl) StartE(
 	// Do not refetch certs if that step already happened once (i.e., we
 	// are restarting a node).
 	if settings.Secure && c.localCertsDir == "" {
+		// Get the certs from the first node.
 		if err := c.RefetchCertsFromNode(ctx, 1); err != nil {
 			return err
 		}
@@ -2279,6 +2279,7 @@ func (c *clusterImpl) StartServiceForVirtualClusterE(
 	}
 
 	if settings.Secure {
+		// Get the certs from the first node.
 		if err := c.RefetchCertsFromNode(ctx, 1); err != nil {
 			return err
 		}
@@ -2336,20 +2337,7 @@ func (c *clusterImpl) RefetchCertsFromNode(ctx context.Context, node int) error
 	// certs. Bypass that distinction (which should be fixed independently, but
 	// that might cause fallout) by using a non-existing dir here.
 	c.localCertsDir = filepath.Join(c.localCertsDir, install.CockroachNodeCertsDir)
-	// Get the certs from the first node.
-	if err := c.Get(ctx, c.l, fmt.Sprintf("./%s", install.CockroachNodeCertsDir), c.localCertsDir, c.Node(node)); err != nil {
-		return errors.Wrap(err, "cluster.StartE")
-	}
-	// Need to prevent world readable files or lib/pq will complain.
-	return filepath.WalkDir(c.localCertsDir, func(path string, d fs.DirEntry, err error) error {
-		if err != nil {
-			return errors.Wrap(err, "walking localCertsDir failed")
-		}
-		if d.IsDir() {
-			return nil
-		}
-		return os.Chmod(path, 0600)
-	})
+	return roachprod.FetchCertsDir(ctx, c.l, c.MakeNodes(c.Node(node)), fmt.Sprintf("./%s", install.CockroachNodeCertsDir), c.localCertsDir)
 }
 
 func (c *clusterImpl) SetDefaultVirtualCluster(name string) {

pkg/roachprod/roachprod.go

@@ -11,6 +11,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"io/fs"
 	"net"
 	"net/http"
 	"net/url"
@@ -3177,3 +3178,29 @@ func FetchLogs(
 			return errors.Wrap(c.Get(ctx, l, c.Nodes, "logs/redacted/combined.log" /* src */, dest), "cluster.FetchLogs")
 		})
 }
+
+// FetchCertsDir downloads the certs directory from the cluster using `roachprod get`
+// and ensures the files are not world readable.
+func FetchCertsDir(
+	ctx context.Context, l *logger.Logger, clusterName, certsDir, destinationDir string,
+) error {
+	c, err := GetClusterFromCache(l, clusterName)
+	if err != nil {
+		return err
+	}
+
+	if err = c.Get(ctx, l, c.Nodes, certsDir, destinationDir); err != nil {
+		return err
+	}
+
+	// Need to prevent world readable files or lib/pq will complain.
+	return filepath.WalkDir(destinationDir, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return errors.Wrap(err, "walking localCertsDir failed")
+		}
+		if d.IsDir() {
+			return nil
+		}
+		return os.Chmod(path, 0600)
+	})
+}