move wg.Add outside of gofunc in setup_e2e_test

Change-Id: Ib01c7af26730c4e93706c31adf79d51b6b6b4990

Author: mattcary

test/e2e/tests/setup_e2e_test.go

@@ -120,8 +120,8 @@ var _ = BeforeSuite(func() {
 				tcc <- NewDefaultTestContext(curZone, strconv.Itoa(randInt))
 			}(zone, j)
 		}
+		wg.Add(1)
 		go func(curZone string) {
-			wg.Add(1)
 			defer GinkgoRecover()
 			defer wg.Done()
 			hdtcc <- NewTestContext(curZone, *hdMinCpuPlatform, *hdMachineType, "0")

test/remote/instance.go

@@ -248,9 +248,9 @@ func (i *InstanceInfo) CreateOrGetInstance(localSSDCount int) error {
 			i.externalIP = externalIP
 		}
 
-		if sshOut, err := i.SSHCheckAlive(); err != nil {
+		if err := i.SSHCheckAlive(); err != nil {
 			err = fmt.Errorf("Instance %v in state RUNNING but not available by SSH: %v", i.cfg.Name, err.Error())
-			klog.Warningf("SSH encountered an error: %v, output: %v", err, sshOut)
+			klog.Warningf("SSH encountered an error: %v", err)
 			return false, nil
 		}
 		klog.V(4).Infof("Instance %v in state RUNNING and available by SSH", i.cfg.Name)

test/remote/ssh.go

@@ -22,7 +22,9 @@ import (
 	"os/exec"
 	"os/user"
 	"strings"
+	"time"
 
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/klog/v2"
 )
 
@@ -88,8 +90,14 @@ func (i *InstanceInfo) SSHNoSudo(cmd ...string) (string, error) {
 }
 
 // SSHCheckAlive just pings the server quickly to check whether it is reachable by SSH
-func (i *InstanceInfo) SSHCheckAlive() (string, error) {
-	return runSSHCommand("ssh", []string{i.GetSSHTarget(), "-o", "ConnectTimeout=10", "--", "echo"}...)
+func (i *InstanceInfo) SSHCheckAlive() error {
+	return wait.Poll(5*time.Second, time.Minute, func() (bool, error) {
+		out, err := runSSHCommand("ssh", []string{i.GetSSHTarget(), "-o", "ConnectTimeout=10", "--", "echo"}...)
+		if err != nil {
+			klog.V(2).Infof("ssh error, retrying: %v, %s", err, out)
+		}
+		return err == nil, nil
+	})
 }
 
 // runSSHCommand executes the ssh or scp command, adding the flag provided --ssh-options