fix(ec2): increase robustness of checking ssm agent ping status. (aws#6621)

## Problem
As part of the EC2 Connect process, we check if the SSM agent is
pingable on the target instance. This is done here:
https://github.com/aws/aws-toolkit-vscode/blob/b9af56c3097242fb796995479f864d95098bf713/packages/core/src/awsService/ec2/model.ts#L153-L162

This check can fail, and is currently does fail a decent amount of the
time in telemetry.

## Solution
- wrap the check in a `waitUntil`, retrying every half second up to 10
times.

---

- Treat all work as PUBLIC. Private `feature/x` branches will not be
squash-merged at release time.
- Your code changes must meet the guidelines in
[CONTRIBUTING.md](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#guidelines).
- License: I confirm that my contribution is made under the terms of the
Apache 2.0 license.

Author: Hweinstock

packages/core/src/awsService/ec2/model.ts

@@ -29,7 +29,7 @@ import {
     testSshConnection,
 } from '../../shared/extensions/ssh'
 import { getLogger } from '../../shared/logger/logger'
-import { CancellationError, Timeout } from '../../shared/utilities/timeoutUtils'
+import { CancellationError, Timeout, waitUntil } from '../../shared/utilities/timeoutUtils'
 import { showMessageWithCancel } from '../../shared/utilities/messages'
 import { SshConfig } from '../../shared/sshConfig'
 import { SshKeyPair } from './sshKeyPair'
@@ -150,8 +150,14 @@ export class Ec2Connecter implements vscode.Disposable {
         }
     }
 
-    private async checkForInstanceSsmError(selection: Ec2Selection): Promise<void> {
-        const isSsmAgentRunning = (await this.ssmClient.getInstanceAgentPingStatus(selection.instanceId)) === 'Online'
+    public async checkForInstanceSsmError(
+        selection: Ec2Selection,
+        options?: Partial<{ interval: number; timeout: number }>
+    ): Promise<void> {
+        const isSsmAgentRunning = await waitUntil(
+            async () => (await this.ssmClient.getInstanceAgentPingStatus(selection.instanceId)) === 'Online',
+            { interval: options?.interval ?? 500, timeout: options?.timeout ?? 5000 }
+        )
 
         if (!isSsmAgentRunning) {
             this.throwConnectionError('Is SSM Agent running on the target instance?', selection, {

packages/core/src/test/awsService/ec2/model.test.ts

@@ -125,6 +125,17 @@ describe('Ec2ConnectClient', function () {
             }
         })
 
+        it('retries if agent status is not online', async function () {
+            const instanceAgentStatus = sinon.stub(SsmClient.prototype, 'getInstanceAgentPingStatus')
+            instanceAgentStatus.onFirstCall().resolves('Offline')
+            instanceAgentStatus.onSecondCall().resolves('Online')
+            try {
+                await client.checkForInstanceSsmError(instanceSelection, { interval: 10, timeout: 100 })
+            } catch (err) {
+                assert.ok(false, `checkForInstanceSsmError failed with error '${err}'`)
+            }
+        })
+
         it('does not throw an error if all checks pass', async function () {
             sinon.stub(Ec2Connecter.prototype, 'isInstanceRunning').resolves(true)
             sinon.stub(Ec2Connecter.prototype, 'getAttachedIamRole').resolves({ Arn: 'testRole' } as IAM.Role)