ci: add retries to test setup in case the Atlas Dev environment is unstable

tests/integration/tools/atlas/accessLists.test.ts

@@ -1,6 +1,6 @@
-import { describeWithAtlas, withProject } from "./atlasHelpers.js";
+import { afterAllWithRetry, beforeAllWithRetry, describeWithAtlas, withProject } from "./atlasHelpers.js";
 import { expectDefined, getResponseElements } from "../../helpers.js";
-import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { describe, expect, it } from "vitest";
 import { ensureCurrentIpInAccessList } from "../../../../src/common/atlas/accessListUtils.js";
 
 function generateRandomIp(): string {
@@ -17,13 +17,13 @@ describeWithAtlas("ip access lists", (integration) => {
         const cidrBlocks = [generateRandomIp() + "/16", generateRandomIp() + "/24"];
         const values = [...ips, ...cidrBlocks];
 
-        beforeAll(async () => {
+        beforeAllWithRetry(async () => {
             const apiClient = integration.mcpServer().session.apiClient;
             const ipInfo = await apiClient.getIpInfo();
             values.push(ipInfo.currentIpv4Address);
         });
 
-        afterAll(async () => {
+        afterAllWithRetry(async () => {
             const apiClient = integration.mcpServer().session.apiClient;
 
             const projectId = getProjectId();

tests/integration/tools/atlas/atlasHelpers.ts

@@ -4,7 +4,7 @@ import type { ApiClient } from "../../../../src/common/atlas/apiClient.js";
 import type { IntegrationTest } from "../../helpers.js";
 import { setupIntegrationTest, defaultTestConfig, defaultDriverOptions } from "../../helpers.js";
 import type { SuiteCollector } from "vitest";
-import { afterAll, beforeAll, describe } from "vitest";
+import { beforeAll, afterAll, describe, it } from "vitest";
 
 export type IntegrationTestFunction = (integration: IntegrationTest) => void;
 
@@ -36,19 +36,13 @@ export function withProject(integration: IntegrationTest, fn: ProjectTestFunctio
     return describe("with project", () => {
         let projectId: string = "";
 
-        beforeAll(async () => {
+        beforeAllWithRetry(async () => {
             const apiClient = integration.mcpServer().session.apiClient;
-
-            try {
-                const group = await createProject(apiClient);
-                projectId = group.id;
-            } catch (error) {
-                console.error("Failed to create project:", error);
-                throw error;
-            }
+            const group = await createProject(apiClient);
+            projectId = group.id;
         });
 
-        afterAll(async () => {
+        afterAllWithRetry(async () => {
             const apiClient = integration.mcpServer().session.apiClient;
             if (projectId) {
                 // projectId may be empty if beforeAll failed.
@@ -70,6 +64,90 @@ export function withProject(integration: IntegrationTest, fn: ProjectTestFunctio
     });
 }
 
+const MAX_ATLAS_STEP_ATTEMPTS = 10;
+const SETUP_BACKOFF_MS = 10;
+
+export function beforeAllWithRetry(fixture: () => Promise<void>): void {
+    beforeAll(async () => {
+        let lastError: Error | undefined = undefined;
+
+        for (let attempt = 0; attempt < MAX_ATLAS_STEP_ATTEMPTS; attempt++) {
+            try {
+                await fixture();
+                lastError = undefined;
+                break;
+            } catch (error: unknown) {
+                if (error instanceof Error) {
+                    lastError = error;
+                } else {
+                    lastError = new Error(String(error));
+                }
+
+                console.error("beforeAll(attempt:", attempt, "):", error);
+                await new Promise((resolve) => setTimeout(resolve, SETUP_BACKOFF_MS * attempt));
+            }
+        }
+
+        if (lastError) {
+            throw lastError;
+        }
+    });
+}
+
+export function afterAllWithRetry(fixture: () => Promise<void>): void {
+    afterAll(async () => {
+        let lastError: Error | undefined = undefined;
+
+        for (let attempt = 0; attempt < MAX_ATLAS_STEP_ATTEMPTS; attempt++) {
+            try {
+                await fixture();
+                lastError = undefined;
+                break;
+            } catch (error) {
+                if (error instanceof Error) {
+                    lastError = error;
+                } else {
+                    lastError = new Error(String(error));
+                }
+                console.error("afterAll(attempt:", attempt, "):", error);
+                await new Promise((resolve) => setTimeout(resolve, SETUP_BACKOFF_MS * attempt));
+            }
+        }
+
+        if (lastError) {
+            throw lastError;
+        }
+    });
+}
+
+export function itWithRetry(name: string, test: () => Promise<void>): void {
+    // complains about not having assertions, but assertions are inside the test function
+    // eslint-disable-next-line
+    it(name, async () => {
+        let lastError: Error | undefined = undefined;
+
+        for (let attempt = 0; attempt < MAX_ATLAS_STEP_ATTEMPTS; attempt++) {
+            try {
+                await test();
+                lastError = undefined;
+                break;
+            } catch (error) {
+                if (error instanceof Error) {
+                    lastError = error;
+                } else {
+                    lastError = new Error(String(error));
+                }
+                console.error(`${name} (attempt: ${attempt}):`, error);
+                await new Promise((resolve) => setTimeout(resolve, SETUP_BACKOFF_MS * attempt));
+            }
+        }
+
+        if (lastError) {
+            throw lastError;
+        }
+    });
+}
+
 export function parseTable(text: string): Record<string, string>[] {
     const data = text
         .split("\n")

tests/integration/tools/atlas/clusters.test.ts

@@ -1,8 +1,8 @@
 import type { Session } from "../../../../src/common/session.js";
 import { expectDefined, getResponseElements } from "../../helpers.js";
-import { describeWithAtlas, withProject, randomId } from "./atlasHelpers.js";
+import { describeWithAtlas, withProject, randomId, afterAllWithRetry, beforeAllWithRetry } from "./atlasHelpers.js";
 import type { ClusterDescription20240805 } from "../../../../src/common/atlas/openapi.js";
-import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { describe, expect, it } from "vitest";
 
 function sleep(ms: number): Promise<void> {
     return new Promise((resolve) => setTimeout(resolve, ms));
@@ -60,7 +60,7 @@ describeWithAtlas("clusters", (integration) => {
     withProject(integration, ({ getProjectId }) => {
         const clusterName = "ClusterTest-" + randomId;
 
-        afterAll(async () => {
+        afterAllWithRetry(async () => {
             const projectId = getProjectId();
             if (projectId) {
                 const session: Session = integration.mcpServer().session;
@@ -160,7 +160,7 @@ describeWithAtlas("clusters", (integration) => {
         });
 
         describe("atlas-connect-cluster", () => {
-            beforeAll(async () => {
+            beforeAllWithRetry(async () => {
                 const projectId = getProjectId();
                 await waitCluster(integration.mcpServer().session, projectId, clusterName, (cluster) => {
                     return (

tests/integration/tools/atlas/projects.test.ts

@@ -1,14 +1,14 @@
 import { ObjectId } from "mongodb";
-import { parseTable, describeWithAtlas } from "./atlasHelpers.js";
+import { parseTable, describeWithAtlas, afterAllWithRetry, itWithRetry } from "./atlasHelpers.js";
 import { expectDefined, getDataFromUntrustedContent, getResponseElements } from "../../helpers.js";
-import { afterAll, describe, expect, it } from "vitest";
+import { describe, expect } from "vitest";
 
 const randomId = new ObjectId().toString();
 
 describeWithAtlas("projects", (integration) => {
     const projName = "testProj-" + randomId;
 
-    afterAll(async () => {
+    afterAllWithRetry(async () => {
         const session = integration.mcpServer().session;
 
         const projects = await session.apiClient.listProjects();
@@ -27,7 +27,7 @@
     });
 
     describe("atlas-create-project", () => {
-        it("should have correct metadata", async () => {
+        itWithRetry("should have correct metadata", async () => {
             const { tools } = await integration.mcpClient().listTools();
             const createProject = tools.find((tool) => tool.name === "atlas-create-project");
             expectDefined(createProject);
@@ -36,7 +36,7 @@
             expect(createProject.inputSchema.properties).toHaveProperty("projectName");
             expect(createProject.inputSchema.properties).toHaveProperty("organizationId");
         });
-        it("should create a project", async () => {
+        itWithRetry("should create a project", async () => {
             const response = await integration.mcpClient().callTool({
                 name: "atlas-create-project",
                 arguments: { projectName: projName },
@@ -44,11 +44,11 @@
 
             const elements = getResponseElements(response);
             expect(elements).toHaveLength(1);
             expect(elements[0]?.text).toContain(projName);
         });
     });
     describe("atlas-list-projects", () => {
-        it("should have correct metadata", async () => {
+        itWithRetry("should have correct metadata", async () => {
             const { tools } = await integration.mcpClient().listTools();
             const listProjects = tools.find((tool) => tool.name === "atlas-list-projects");
             expectDefined(listProjects);
@@ -57,11 +57,10 @@
             expect(listProjects.inputSchema.properties).toHaveProperty("orgId");
         });
 
-        it("returns project names", async () => {
+        itWithRetry("returns project names", async () => {
             const response = await integration.mcpClient().callTool({ name: "atlas-list-projects", arguments: {} });
             const elements = getResponseElements(response);
-            expect(elements).toHaveLength(2);
             expect(elements[0]?.text).toMatch(/Found \d+ projects/);
             expect(elements[1]?.text).toContain("<untrusted-user-data-");
             expect(elements[1]?.text).toContain(projName);
             const data = parseTable(getDataFromUntrustedContent(elements[1]?.text ?? ""));