roachprod: add retries to Install

This change adds retries to `roachprod.Install` to mitigate
test failures due to package installation failures.

The webhook test in `cdc.go` is updated to use `Cluster.Install` to
install `go`.

The install command for `go` is just `sudo apt --yes install golang-go;`
now. No previously existing uses of `Cluster.Install` used this command,
so changing it should be safe.

Informs: cockroachdb#103316
Informs: cockroachdb#107088
Closes: cockroachdb#71934
Release note: None
Epic: None

Author: jayshrivastava

pkg/cmd/roachtest/tests/cdc.go

@@ -91,6 +91,16 @@ type cdcTester struct {
 	doneCh     chan struct{}
 }
 
+// The node on which the webhook sink will be installed and run on.
+func (ct *cdcTester) webhookSinkNode() option.NodeListOption {
+	return ct.cluster.Node(ct.cluster.Spec().NodeCount)
+}
+
+// The node on which the kafka sink will be installed and run on.
+func (ct *cdcTester) kafkaSinkNode() option.NodeListOption {
+	return ct.cluster.Node(ct.cluster.Spec().NodeCount)
+}
+
 // startStatsCollection sets the start point of the stats collection window
 // and returns a function which should be called at the end of the test to dump a
 // stats.json file to the artifacts directory.
@@ -158,7 +168,7 @@ func (ct *cdcTester) setupSink(args feedArgs) string {
 		sinkURI = `experimental-gs://cockroach-tmp/roachtest/` + ts + "?AUTH=implicit"
 	case webhookSink:
 		ct.t.Status("webhook install")
-		webhookNode := ct.cluster.Node(ct.cluster.Spec().NodeCount)
+		webhookNode := ct.webhookSinkNode()
 		rootFolder := `/home/ubuntu`
 		nodeIPs, _ := ct.cluster.ExternalIP(ct.ctx, ct.logger, webhookNode)
 
@@ -184,21 +194,6 @@ func (ct *cdcTester) setupSink(args feedArgs) string {
 			ct.t.Fatal(err)
 		}
 
-		// As seen in #107061, this can hit a 503 Service Unavailable when
-		// trying to download the package, so we retry every 30 seconds
-		// for up to 5 mins below.
-		err = retry.WithMaxAttempts(ct.ctx, retry.Options{
-			InitialBackoff: 30 * time.Second,
-			Multiplier:     1,
-		}, 10, func() error {
-			err = ct.cluster.RunE(ct.ctx, webhookNode, `sudo apt --yes install golang-go;`)
-			err = errors.Wrap(err, "infrastructure failure; could not install golang")
-			return err
-		})
-		if err != nil {
-			ct.t.Skip(err)
-		}
-
 		// Start the server in its own monitor to not block ct.mon.Wait()
 		serverExecCmd := fmt.Sprintf(`go run webhook-server-%d.go`, webhookPort)
 		m := ct.cluster.NewMonitor(ct.ctx, ct.workloadNode)
@@ -219,7 +214,7 @@ func (ct *cdcTester) setupSink(args feedArgs) string {
 	case pubsubSink:
 		sinkURI = changefeedccl.GcpScheme + `://cockroach-ephemeral` + "?AUTH=implicit&topic_name=pubsubSink-roachtest&region=us-east1"
 	case kafkaSink:
-		kafkaNode := ct.cluster.Node(ct.cluster.Spec().NodeCount)
+		kafkaNode := ct.kafkaSinkNode()
 		kafka := kafkaManager{
 			t:     ct.t,
 			c:     ct.cluster,
@@ -1303,6 +1298,13 @@ func registerCDC(r registry.Registry) {
 			ct := newCDCTester(ctx, t, c)
 			defer ct.Close()
 
+			// Consider an installation failure to be a flake which is out of
+			// our control. This should be rare.
+			err := c.Install(ctx, t.L(), ct.webhookSinkNode(), "go")
+			if err != nil {
+				t.Skip(err)
+			}
+
 			ct.runTPCCWorkload(tpccArgs{warehouses: 100, duration: "30m"})
 
 			// The deprecated webhook sink is unable to handle the throughput required for 100 warehouses
@@ -1351,7 +1353,7 @@ func registerCDC(r registry.Registry) {
 
 			ct.runTPCCWorkload(tpccArgs{warehouses: 1})
 
-			kafkaNode := ct.cluster.Node(ct.cluster.Spec().NodeCount)
+			kafkaNode := ct.kafkaSinkNode()
 			kafka := kafkaManager{
 				t:     ct.t,
 				c:     ct.cluster,

pkg/roachprod/BUILD.bazel

@@ -25,6 +25,7 @@ go_library(
         "//pkg/server/debug/replay",
         "//pkg/util/ctxgroup",
         "//pkg/util/httputil",
+        "//pkg/util/retry",
         "//pkg/util/syncutil",
         "//pkg/util/timeutil",
         "@com_github_cockroachdb_errors//:errors",

pkg/roachprod/install/install.go

@@ -102,15 +102,7 @@ sudo apt-get update;
 sudo apt-get install -y gcc;
 `,
 
-	// graphviz and rlwrap are useful for pprof
-	"go": `
-sudo apt-get update;
-sudo apt-get install -y graphviz rlwrap;
-
-curl https://dl.google.com/go/go1.12.linux-amd64.tar.gz | sudo tar -C /usr/local -xz;
-echo 'export PATH=$PATH:/usr/local/go/bin' | sudo tee /etc/profile.d/go.sh > /dev/null;
-sudo chmod +x /etc/profile.d/go.sh;
-`,
+	"go": `sudo apt --yes install golang-go;`,
 
 	"haproxy": `
 sudo apt-get update;

pkg/roachprod/roachprod.go

@@ -45,6 +45,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/server/debug/replay"
 	"github.com/cockroachdb/cockroach/pkg/util/ctxgroup"
 	"github.com/cockroachdb/cockroach/pkg/util/httputil"
+	"github.com/cockroachdb/cockroach/pkg/util/retry"
 	"github.com/cockroachdb/cockroach/pkg/util/syncutil"
 	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
 	"github.com/cockroachdb/errors"
@@ -833,7 +834,21 @@ func Install(ctx context.Context, l *logger.Logger, clusterName string, software
 	if err != nil {
 		return err
 	}
-	return install.Install(ctx, l, c, software)
+
+	// As seen in #103316, this can hit a 503 Service Unavailable when
+	// trying to download the package, so we retry every 30 seconds
+	// for up to 5 mins below. The caller may choose to fail or skip the test.
+	return retry.WithMaxAttempts(ctx, retry.Options{
+		InitialBackoff: 30 * time.Second,
+		Multiplier:     1,
+	}, 10, func() error {
+		err := install.Install(ctx, l, c, software)
+		err = errors.Wrapf(err, "retryable infrastructure error: could not install %s", software)
+		if err != nil {
+			l.Printf(err.Error())
+		}
+		return err
+	})
 }
 
 // Download downloads 3rd party tools, using a GCS cache if possible.