[cloud] Add retry when trying to connect to cloud vm for first time (#581)

## Summary

`copyConfigFileToVM` is the first command we run on the VM. Sometimes is takes fly.io a few seconds to propagate DNS, especially if the VM is located in a different region than the proxy (this can happen if the gateway is in a different region to proxy) We retry a few times to avoid failing the command.

## How was it tested?

`DEVBOX_DEBUG=1 devbox cloud shell`

many times

Author: mikeland73

internal/cloud/cloud.go

@@ -338,7 +338,11 @@ func copyConfigFileToVM(hostname, username, projectDir, pathInVM string) error {
 
 	// Ensure the devbox-project's directory exists in the VM
 	mkdirCmd := openssh.Command(username, hostname)
-	_, err := mkdirCmd.ExecRemote(fmt.Sprintf(`mkdir -p "%s"`, pathInVM))
+	// This is the first command we run on the VM. Sometimes is takes fly.io a few seconds
+	// to propagate DNS, especially if the VM is located in a different region than
+	// the proxy (this can happen if the gateway is in a different region to proxy)
+	// We retry a few times to avoid failing the command.
+	_, err := mkdirCmd.ExecRemoteWithRetry(fmt.Sprintf(`mkdir -p "%s"`, pathInVM), 5, 4)
 	if err != nil {
 		debug.Log("error copying config file to VM: %v", err)
 		return errors.WithStack(err)

internal/cloud/openssh/cmd.go

@@ -8,11 +8,13 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
+	"math"
 	"net"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strconv"
+	"time"
 
 	"go.jetpack.io/devbox/internal/debug"
 )
@@ -66,6 +68,23 @@ func (c *Cmd) ExecRemote(cmd string) ([]byte, error) {
 	return stdout.Bytes(), nil
 }
 
+// ExecRemoteWithRetry runs the given command on the remote host, retrying
+// with an exponential backoff if the command fails. maxWait is the maximum
+// seconds we wait in between retries.
+func (c *Cmd) ExecRemoteWithRetry(cmd string, retries, maxWait int) ([]byte, error) {
+	var err error
+	var stdout []byte
+	for i := 0; i < (retries + 1); i++ {
+		if stdout, err = c.ExecRemote(cmd); err == nil {
+			break
+		}
+		wait := int(math.Min(float64(maxWait), math.Pow(2, float64(i))))
+		debug.Log("Error: %v Retrying ExecRemote in %d seconds", err, wait)
+		time.Sleep(time.Duration(wait) * time.Second)
+	}
+	return stdout, err
+}
+
 func (c *Cmd) cmd(sshArgs ...string) *exec.Cmd {
 	host, port := splitHostPort(c.DestinationAddr)
 	cmd := exec.Command("ssh", "-l", c.Username)