[sshshim] re-try 3 times for finding activeVM (#554)

## Summary

Its possible the check for activeVM failed due to some ephemeral reason.
So, we
try 3 times with a sleep of 10 seconds in between.

## How was it tested?

1. verified that sync still works.

2. 
- killed the VM
- tailed the logs at `~/.config/devbox/ssh/shims/logs.txt`
```
[DEBUG] 2023/01/27 17:09:59 /Users/savil/code/jetpack/devbox/internal/cloud/openssh/sshshim/logger.go:27: Debug mode enabled.
[DEBUG] 2023/01/27 17:09:59 /Users/savil/code/jetpack/devbox/internal/cloud/openssh/sshshim/logger.go:28: started sshshim
[DEBUG] 2023/01/27 17:09:59 /Users/savil/code/jetpack/devbox/internal/cloud/openssh/sshshim/command.go:30: os.Args: [/Users/savil/.config/devbox/ssh/shims/ssh -oConnectTimeout=5 -oServerAliveInterval=10 -oServerAliveCountMax=1 [email protected] .mutagen/agents/0.16.2/mutagen-agent synchronizer --log-level=info]
[DEBUG] 2023/01/27 17:09:59 /Users/savil/code/jetpack/devbox/internal/cloud/openssh/sshshim/mutagen.go:20: Found vmAddr: [email protected]
[DEBUG] 2023/01/27 17:09:59 /Users/savil/code/jetpack/devbox/internal/cloud/openssh/sshshim/logger.go:27: Debug mode enabled.
[DEBUG] 2023/01/27 17:09:59 /Users/savil/code/jetpack/devbox/internal/cloud/openssh/sshshim/logger.go:28: started sshshim
[DEBUG] 2023/01/27 17:09:59 /Users/savil/code/jetpack/devbox/internal/cloud/openssh/sshshim/command.go:30: os.Args: [/Users/savil/.config/devbox/ssh/shims/ssh -oConnectTimeout=5 -oServerAliveInterval=10 -oServerAliveCountMax=1 [email protected] .mutagen/agents/0.16.2/mutagen-agent synchronizer --log-level=info]
[DEBUG] 2023/01/27 17:09:59 /Users/savil/code/jetpack/devbox/internal/cloud/openssh/sshshim/mutagen.go:20: Found vmAddr: [email protected]
[DEBUG] 2023/01/27 17:10:00 /Users/savil/code/jetpack/devbox/internal/cloud/openssh/sshshim/mutagen.go:85: checkActiveVM: No active VM. returning false for exit status 255
[DEBUG] 2023/01/27 17:10:00 /Users/savil/code/jetpack/devbox/internal/cloud/openssh/sshshim/mutagen.go:67: Try 1 failed to find activeVM for e784935c246683.vm.devbox-vms.internal
[DEBUG] 2023/01/27 17:10:00 /Users/savil/code/jetpack/devbox/internal/cloud/openssh/sshshim/mutagen.go:85: checkActiveVM: No active VM. returning false for exit status 255
[DEBUG] 2023/01/27 17:10:00 /Users/savil/code/jetpack/devbox/internal/cloud/openssh/sshshim/mutagen.go:67: Try 2 failed to find activeVM for e784935c246683.vm.devbox-vms.internal
[DEBUG] 2023/01/27 17:10:00 /Users/savil/code/jetpack/devbox/internal/cloud/openssh/sshshim/mutagen.go:31: terminating mutagen session for vm: e784935c246683.vm.devbox-vms.internal
[DEBUG] 2023/01/27 17:10:00 /Users/savil/code/jetpack/devbox/internal/cloud/mutagen/wrapper.go:176: running mutagen cmd /Users/savil/.cache/mutagen/bin/mutagen sync terminate --label-selector devbox-vm=e784935c246683 with MUTAGEN env: , MUTAGEN_SSH_PATH=/Users/savil/.config/devbox/ssh/shims, MUTAGEN_DATA_DIRECTORY=/Users/savil/.config/devbox/mutagen
[DEBUG] 2023/01/27 17:10:00 /Users/savil/code/jetpack/devbox/internal/cloud/mutagen/wrapper.go:164: execMutagen worked for cmd: /Users/savil/.cache/mutagen/bin/mutagen sync terminate --label-selector devbox-vm=e784935c246683
[DEBUG] 2023/01/27 17:10:00 /Users/savil/code/jetpack/devbox/internal/cloud/mutagen/wrapper.go:176: running mutagen cmd /Users/savil/.cache/mutagen/bin/mutagen forward terminate --label-selector devbox=true with MUTAGEN env: , MUTAGEN_DATA_DIRECTORY=/Users/savil/.config/devbox/mutagen, MUTAGEN_SSH_PATH=/Users/savil/.config/devbox/ssh/shims
[DEBUG] 2023/01/27 17:10:00 /Users/savil/code/jetpack/devbox/internal/cloud/mutagen/wrapper.go:164: execMutagen worked for cmd: /Users/savil/.cache/mutagen/bin/mutagen forward terminate --label-selector devbox=true
```
NOTE: output is prior to my adding Sleep. I'll re-check with the sleep,
but sending PR for review. (UPDATE: confirmed, it works as expected with
Sleep as well).

Author: savil

internal/cloud/openssh/sshshim/mutagen.go

@@ -25,7 +25,7 @@ func EnsureLiveVMOrTerminateMutagenSessions(sshArgs []string) (bool, error) {
 		return true, nil
 	}
 
-	if isActive, err := checkActiveVM(vmAddr); err != nil {
+	if isActive, err := checkActiveVMWithRetries(vmAddr); err != nil {
 		return false, errors.WithStack(err)
 	} else if !isActive {
 		debug.Log("terminating mutagen session for vm: %s", vmAddr)
@@ -54,6 +54,23 @@ func terminateMutagenSessions(vmAddr string) error {
 	return mutagenbox.ForwardTerminateAll()
 }
 
+func checkActiveVMWithRetries(vmAddr string) (bool, error) {
+	var err error
+
+	// Try 3 times:
+	for num := 0; num < 3; num++ {
+		var isActive bool
+		isActive, err = checkActiveVM(vmAddr)
+		if err == nil && isActive {
+			// found an active VM
+			return true, nil
+		}
+		time.Sleep(10 * time.Second)
+		debug.Log("Try %d failed to find activeVM for %s", num, vmAddr)
+	}
+	return false, err
+}
+
 func checkActiveVM(vmAddr string) (bool, error) {
 
 	ctx, cancel := context.WithTimeout(context.Background(), time.Minute*2)