Added retry mechanism to catch known errors for health checks (#2209)

* Added retry mechanism to catch known errors

* Updated the error message to cover multiple scenarios

* Added Retry Mechanism that can work for any tests

* Update .changeset/nine-toes-dress.md

Co-authored-by: Kamil Sobol <[email protected]>

* chore:Update API

* Refined retry mechanism removed timeout and delay

* chore: Update API

* chore: update API

* Updated error check for test case

* Updated retry test case condition

* updated the maxretry parameter

* chore: update changeset

* chore: update changeset

* Updated a test case without retry

* Update packages/integration-tests/src/amplify_app_pool.ts

Co-authored-by: Kamil Sobol <[email protected]>

* updated retry mechanism

---------

Co-authored-by: Kamil Sobol <[email protected]>

Author: vigy02

.changeset/nine-toes-dress.md

@@ -0,0 +1,2 @@
+---
+---

.eslint_dictionary.json

@@ -136,6 +136,7 @@
   "renderer",
   "repo",
   "resolvers",
+  "retryable",
   "saml",
   "scala",
   "schema",

packages/integration-tests/src/amplify_app_pool.ts

@@ -13,6 +13,7 @@ import {
 } from '@aws-sdk/client-amplify';
 import { shortUuid } from './short_uuid.js';
 import { e2eToolingClientConfig } from './e2e_tooling_client_config.js';
+import { runWithRetry } from './retry.js';
 
 export type TestBranch = {
   readonly appId: string;
@@ -46,42 +47,46 @@ class DefaultAmplifyAppPool implements AmplifyAppPool {
   }
 
   fetchTestBranchDetails = async (testBranch: TestBranch): Promise<Branch> => {
-    const branch = (
-      await this.amplifyClient.send(
-        new GetBranchCommand({
-          appId: testBranch.appId,
-          branchName: testBranch.branchName,
-        })
-      )
-    ).branch;
-    if (!branch) {
-      throw new Error(
-        `Failed to retrieve ${testBranch.branchName} branch of app ${testBranch.appId}`
-      );
-    }
-    return branch;
+    return this.retryableOperation(async () => {
+      const branch = (
+        await this.amplifyClient.send(
+          new GetBranchCommand({
+            appId: testBranch.appId,
+            branchName: testBranch.branchName,
+          })
+        )
+      ).branch;
+      if (!branch) {
+        throw new Error(
+          `Failed to retrieve ${testBranch.branchName} branch of app ${testBranch.appId}`
+        );
+      }
+      return branch;
+    });
   };
 
   createTestBranch = async (): Promise<TestBranch> => {
-    const app = await this.getAppWithCapacity();
-    const branch = (
-      await this.amplifyClient.send(
-        new CreateBranchCommand({
-          branchName: `${this.testBranchPrefix}${shortUuid()}`,
+    return this.retryableOperation(async () => {
+      const app = await this.getAppWithCapacity();
+      const branch = (
+        await this.amplifyClient.send(
+          new CreateBranchCommand({
+            branchName: `${this.testBranchPrefix}${shortUuid()}`,
+            appId: app.appId,
+          })
+        )
+      ).branch;
+      if (app.appId && branch?.branchName) {
+        const testBranch: TestBranch = {
           appId: app.appId,
-        })
-      )
-    ).branch;
-    if (app.appId && branch?.branchName) {
-      const testBranch: TestBranch = {
-        appId: app.appId,
-        branchName: branch.branchName,
-      };
-      this.branchesCreated.push(testBranch);
-      return testBranch;
-    }
+          branchName: branch.branchName,
+        };
+        this.branchesCreated.push(testBranch);
+        return testBranch;
+      }
 
-    throw new Error('Unable to create branch');
+      throw new Error('Unable to create branch');
+    });
   };
 
   private listAllTestAmplifyApps = async (): Promise<Array<App>> => {
@@ -173,6 +178,16 @@ class DefaultAmplifyAppPool implements AmplifyAppPool {
       }
     }
   };
+
+  private retryableOperation = <T>(operation: () => Promise<T>) => {
+    return runWithRetry(operation, (error) => {
+      // Add specific error conditions here that warrant a retry
+      return (
+        error.message.includes('Unexpected token') ||
+        error.message.includes('Bad control character')
+      );
+    });
+  };
 }
 
 export const amplifyAppPool: AmplifyAppPool = new DefaultAmplifyAppPool(

packages/integration-tests/src/package_manager_sanity_checks.test.ts

@@ -26,6 +26,7 @@ import {
   runWithPackageManager,
 } from './process-controller/process_controller.js';
 import { amplifyAtTag } from './constants.js';
+import { runWithRetry } from './retry.js';
 
 void describe('getting started happy path', async () => {
   let branchBackendIdentifier: BackendIdentifier;
@@ -81,19 +82,35 @@ void describe('getting started happy path', async () => {
     if (packageManager === 'pnpm' && process.platform === 'win32') {
       return;
     }
-    if (packageManager === 'yarn-classic') {
-      await execa('yarn', ['add', 'create-amplify'], { cwd: tempDir });
-      await execaCommand('./node_modules/.bin/create-amplify --yes --debug', {
-        cwd: tempDir,
-        env: { npm_config_user_agent: 'yarn/1.22.21' },
-      });
-    } else {
-      await runPackageManager(
-        packageManager,
-        ['create', amplifyAtTag, '--yes'],
-        tempDir
-      ).run();
-    }
+
+    await runWithRetry(
+      async () => {
+        if (packageManager === 'yarn-classic') {
+          await execa('yarn', ['add', 'create-amplify'], { cwd: tempDir });
+          await execaCommand(
+            './node_modules/.bin/create-amplify --yes --debug',
+            {
+              cwd: tempDir,
+              env: { npm_config_user_agent: 'yarn/1.22.21' },
+            }
+          );
+        } else {
+          await runPackageManager(
+            packageManager,
+            ['create', amplifyAtTag, '--yes'],
+            tempDir
+          ).run();
+        }
+      },
+      (error) => {
+        // Retry on network-related errors or command failures
+        return (
+          error.message.includes('exit code 1') &&
+          (error.message.includes('aws-amplify') ||
+            error.message.includes('Command failed with exit code 1: yarn add'))
+        );
+      }
+    );
 
     const pathPrefix = path.join(tempDir, 'amplify');
 

packages/integration-tests/src/retry.ts

@@ -0,0 +1,32 @@
+/**
+ * Executes an asynchronous operation with retry logic.
+ * This function attempts to execute the provided callable function multiple times
+ * based on the specified retry conditions. It's useful for handling transient
+ * errors or temporary service unavailability.
+ */
+export const runWithRetry = async <T>(
+  callable: () => Promise<T>,
+  retryPredicate: (error: Error) => boolean,
+  maxAttempts = 3
+): Promise<T> => {
+  const collectedErrors: Error[] = [];
+
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    try {
+      const result = await callable();
+      return result;
+    } catch (error) {
+      if (error instanceof Error) {
+        collectedErrors.push(error);
+        if (!retryPredicate(error)) {
+          throw error;
+        }
+      }
+    }
+  }
+
+  throw new AggregateError(
+    collectedErrors,
+    `All ${maxAttempts} attempts failed`
+  );
+};