Skip to content

chore: fix flaky e2e tests #438

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Sep 30, 2025
Merged

chore: fix flaky e2e tests #438

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7515933
chore: fix agentic chat flaky test
ranst91 Sep 29, 2025
a652833
chore: fix agentic chat flaky test
ranst91 Sep 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 6 additions & 10 deletions typescript-sdk/apps/dojo/e2e/tests/adkMiddlewareTests/agenticChatPage.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,8 @@ test.describe("Agentic Chat Feature", () => {
await chat.agentGreeting.waitFor({ state: "visible" });

// Store initial background color
const initialBackground = await chat.getBackground();
const backgroundContainer = page.locator('[data-testid="background-container"]')
const initialBackground = await backgroundContainer.evaluate(el => getComputedStyle(el).backgroundColor);
console.log("Initial background color:", initialBackground);

// 1. Send message to change background to blue
Expand All @@ -52,8 +53,8 @@ test.describe("Agentic Chat Feature", () => {
);
await waitForAIResponse(page);

const backgroundBlue = await chat.getBackground();
expect(backgroundBlue).not.toBe(initialBackground);
await expect(backgroundContainer).not.toHaveCSS('background-color', initialBackground, { timeout: 7000 });
const backgroundBlue = await backgroundContainer.evaluate(el => getComputedStyle(el).backgroundColor);
// Check if background is blue (string color name or contains blue)
expect(backgroundBlue.toLowerCase()).toMatch(/blue|rgb\(.*,.*,.*\)|#[0-9a-f]{6}/);

Expand All @@ -64,15 +65,10 @@ test.describe("Agentic Chat Feature", () => {
);
await waitForAIResponse(page);

const backgroundPink = await chat.getBackground();
expect(backgroundPink).not.toBe(backgroundBlue);
await expect(backgroundContainer).not.toHaveCSS('background-color', backgroundBlue, { timeout: 7000 });
const backgroundPink = await backgroundContainer.evaluate(el => getComputedStyle(el).backgroundColor);
// Check if background is pink (string color name or contains pink)
expect(backgroundPink.toLowerCase()).toMatch(/pink|rgb\(.*,.*,.*\)|#[0-9a-f]{6}/);

// 3. Reset to default
await chat.sendMessage("Reset the background color");
await chat.assertUserMessageVisible("Reset the background color");
await waitForAIResponse(page);
});
});

Expand Down
16 changes: 6 additions & 10 deletions typescript-sdk/apps/dojo/e2e/tests/crewAITests/agenticChatPage.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@ test("[CrewAI] Agentic Chat changes background on message and reset", async ({
await chat.agentGreeting.waitFor({ state: "visible" });

// Store initial background color
const initialBackground = await chat.getBackground();
const backgroundContainer = page.locator('[data-testid="background-container"]')
const initialBackground = await backgroundContainer.evaluate(el => getComputedStyle(el).backgroundColor);
console.log("Initial background color:", initialBackground);

// 1. Send message to change background to blue
Expand All @@ -50,8 +51,8 @@ test("[CrewAI] Agentic Chat changes background on message and reset", async ({
);
await waitForAIResponse(page);

const backgroundBlue = await chat.getBackground();
expect(backgroundBlue).not.toBe(initialBackground);
await expect(backgroundContainer).not.toHaveCSS('background-color', initialBackground, { timeout: 7000 });
const backgroundBlue = await backgroundContainer.evaluate(el => getComputedStyle(el).backgroundColor);
// Check if background is blue (string color name or contains blue)
expect(backgroundBlue.toLowerCase()).toMatch(/blue|rgb\(.*,.*,.*\)|#[0-9a-f]{6}/);

Expand All @@ -62,15 +63,10 @@ test("[CrewAI] Agentic Chat changes background on message and reset", async ({
);
await waitForAIResponse(page);

const backgroundPink = await chat.getBackground();
expect(backgroundPink).not.toBe(backgroundBlue);
await expect(backgroundContainer).not.toHaveCSS('background-color', backgroundBlue, { timeout: 7000 });
const backgroundPink = await backgroundContainer.evaluate(el => getComputedStyle(el).backgroundColor);
// Check if background is pink (string color name or contains pink)
expect(backgroundPink.toLowerCase()).toMatch(/pink|rgb\(.*,.*,.*\)|#[0-9a-f]{6}/);

// 3. Reset to default
await chat.sendMessage("Reset the background color");
await chat.assertUserMessageVisible("Reset the background color");
await waitForAIResponse(page);
});
});

Expand Down
16 changes: 6 additions & 10 deletions typescript-sdk/apps/dojo/e2e/tests/langgraphFastAPITests/agenticChatPage.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@ test("[LangGraph FastAPI] Agentic Chat changes background on message and reset",
await chat.agentGreeting.waitFor({ state: "visible" });

// Store initial background color
const initialBackground = await chat.getBackground();
const backgroundContainer = page.locator('[data-testid="background-container"]')
const initialBackground = await backgroundContainer.evaluate(el => getComputedStyle(el).backgroundColor);
console.log("Initial background color:", initialBackground);

// 1. Send message to change background to blue
Expand All @@ -50,8 +51,8 @@ test("[LangGraph FastAPI] Agentic Chat changes background on message and reset",
);
await waitForAIResponse(page);

const backgroundBlue = await chat.getBackground();
expect(backgroundBlue).not.toBe(initialBackground);
await expect(backgroundContainer).not.toHaveCSS('background-color', initialBackground, { timeout: 7000 });
const backgroundBlue = await backgroundContainer.evaluate(el => getComputedStyle(el).backgroundColor);
// Check if background is blue (string color name or contains blue)
expect(backgroundBlue.toLowerCase()).toMatch(/blue|rgb\(.*,.*,.*\)|#[0-9a-f]{6}/);

Expand All @@ -62,15 +63,10 @@ test("[LangGraph FastAPI] Agentic Chat changes background on message and reset",
);
await waitForAIResponse(page);

const backgroundPink = await chat.getBackground();
expect(backgroundPink).not.toBe(backgroundBlue);
await expect(backgroundContainer).not.toHaveCSS('background-color', backgroundBlue, { timeout: 7000 });
const backgroundPink = await backgroundContainer.evaluate(el => getComputedStyle(el).backgroundColor);
// Check if background is pink (string color name or contains pink)
expect(backgroundPink.toLowerCase()).toMatch(/pink|rgb\(.*,.*,.*\)|#[0-9a-f]{6}/);

// 3. Reset to default
await chat.sendMessage("Reset the background color");
await chat.assertUserMessageVisible("Reset the background color");
await waitForAIResponse(page);
});
});

Expand Down
11 changes: 6 additions & 5 deletions typescript-sdk/apps/dojo/e2e/tests/langgraphPythonTests/agenticChatPage.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@ test("[LangGraph] Agentic Chat changes background on message and reset", async (
await chat.agentGreeting.waitFor({ state: "visible" });

// Store initial background color
const initialBackground = await chat.getBackground();
const backgroundContainer = page.locator('[data-testid="background-container"]')
const initialBackground = await backgroundContainer.evaluate(el => getComputedStyle(el).backgroundColor);
console.log("Initial background color:", initialBackground);

// 1. Send message to change background to blue
Expand All @@ -50,8 +51,8 @@ test("[LangGraph] Agentic Chat changes background on message and reset", async (
);
await waitForAIResponse(page);

const backgroundBlue = await chat.getBackground();
expect(backgroundBlue).not.toBe(initialBackground);
await expect(backgroundContainer).not.toHaveCSS('background-color', initialBackground, { timeout: 7000 });
const backgroundBlue = await backgroundContainer.evaluate(el => getComputedStyle(el).backgroundColor);
// Check if background is blue (string color name or contains blue)
expect(backgroundBlue.toLowerCase()).toMatch(/blue|rgb\(.*,.*,.*\)|#[0-9a-f]{6}/);

Expand All @@ -62,8 +63,8 @@ test("[LangGraph] Agentic Chat changes background on message and reset", async (
);
await waitForAIResponse(page);

const backgroundPink = await chat.getBackground();
expect(backgroundPink).not.toBe(backgroundBlue);
await expect(backgroundContainer).not.toHaveCSS('background-color', backgroundBlue, { timeout: 7000 });
const backgroundPink = await backgroundContainer.evaluate(el => getComputedStyle(el).backgroundColor);
// Check if background is pink (string color name or contains pink)
expect(backgroundPink.toLowerCase()).toMatch(/pink|rgb\(.*,.*,.*\)|#[0-9a-f]{6}/);
});
Expand Down
11 changes: 6 additions & 5 deletions typescript-sdk/apps/dojo/e2e/tests/langgraphTypescriptTests/agenticChatPage.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@ test("[LangGraph] Agentic Chat changes background on message and reset", async (
await chat.agentGreeting.waitFor({ state: "visible" });

// Store initial background color
const initialBackground = await chat.getBackground();
const backgroundContainer = page.locator('[data-testid="background-container"]')
const initialBackground = await backgroundContainer.evaluate(el => getComputedStyle(el).backgroundColor);
console.log("Initial background color:", initialBackground);

// 1. Send message to change background to blue
Expand All @@ -50,8 +51,8 @@ test("[LangGraph] Agentic Chat changes background on message and reset", async (
);
await waitForAIResponse(page);

const backgroundBlue = await chat.getBackground();
expect(backgroundBlue).not.toBe(initialBackground);
await expect(backgroundContainer).not.toHaveCSS('background-color', initialBackground, { timeout: 7000 });
const backgroundBlue = await backgroundContainer.evaluate(el => getComputedStyle(el).backgroundColor);
// Check if background is blue (string color name or contains blue)
expect(backgroundBlue.toLowerCase()).toMatch(/blue|rgb\(.*,.*,.*\)|#[0-9a-f]{6}/);

Expand All @@ -62,8 +63,8 @@ test("[LangGraph] Agentic Chat changes background on message and reset", async (
);
await waitForAIResponse(page);

const backgroundPink = await chat.getBackground();
expect(backgroundPink).not.toBe(backgroundBlue);
await expect(backgroundContainer).not.toHaveCSS('background-color', backgroundBlue, { timeout: 7000 });
const backgroundPink = await backgroundContainer.evaluate(el => getComputedStyle(el).backgroundColor);
// Check if background is pink (string color name or contains pink)
expect(backgroundPink.toLowerCase()).toMatch(/pink|rgb\(.*,.*,.*\)|#[0-9a-f]{6}/);
});
Expand Down
16 changes: 6 additions & 10 deletions typescript-sdk/apps/dojo/e2e/tests/llamaIndexTests/agenticChatPage.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@ test("[LlamaIndex] Agentic Chat changes background on message and reset", async
await chat.agentGreeting.waitFor({ state: "visible" });

// Store initial background color
const initialBackground = await chat.getBackground();
const backgroundContainer = page.locator('[data-testid="background-container"]')
const initialBackground = await backgroundContainer.evaluate(el => getComputedStyle(el).backgroundColor);
console.log("Initial background color:", initialBackground);

// 1. Send message to change background to blue
Expand All @@ -50,8 +51,8 @@ test("[LlamaIndex] Agentic Chat changes background on message and reset", async
);
await waitForAIResponse(page);

const backgroundBlue = await chat.getBackground();
expect(backgroundBlue).not.toBe(initialBackground);
await expect(backgroundContainer).not.toHaveCSS('background-color', initialBackground, { timeout: 7000 });
const backgroundBlue = await backgroundContainer.evaluate(el => getComputedStyle(el).backgroundColor);
// Check if background is blue (string color name or contains blue)
expect(backgroundBlue.toLowerCase()).toMatch(/blue|rgb\(.*,.*,.*\)|#[0-9a-f]{6}/);

Expand All @@ -62,15 +63,10 @@ test("[LlamaIndex] Agentic Chat changes background on message and reset", async
);
await waitForAIResponse(page);

const backgroundPink = await chat.getBackground();
expect(backgroundPink).not.toBe(backgroundBlue);
await expect(backgroundContainer).not.toHaveCSS('background-color', backgroundBlue, { timeout: 7000 });
const backgroundPink = await backgroundContainer.evaluate(el => getComputedStyle(el).backgroundColor);
// Check if background is pink (string color name or contains pink)
expect(backgroundPink.toLowerCase()).toMatch(/pink|rgb\(.*,.*,.*\)|#[0-9a-f]{6}/);

// 3. Reset to default
await chat.sendMessage("Reset the background color to default");
await chat.assertUserMessageVisible("Reset the background color to default");
await waitForAIResponse(page);
});
});

Expand Down
16 changes: 6 additions & 10 deletions typescript-sdk/apps/dojo/e2e/tests/mastraAgentLocalTests/agenticChatPage.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@ test("[MastraAgentLocal] Agentic Chat changes background on message and reset",
await chat.agentGreeting.waitFor({ state: "visible" });

// Store initial background color
const initialBackground = await chat.getBackground();
const backgroundContainer = page.locator('[data-testid="background-container"]')
const initialBackground = await backgroundContainer.evaluate(el => getComputedStyle(el).backgroundColor);
console.log("Initial background color:", initialBackground);

// 1. Send message to change background to blue
Expand All @@ -50,8 +51,8 @@ test("[MastraAgentLocal] Agentic Chat changes background on message and reset",
);
await waitForAIResponse(page);

const backgroundBlue = await chat.getBackground();
expect(backgroundBlue).not.toBe(initialBackground);
await expect(backgroundContainer).not.toHaveCSS('background-color', initialBackground, { timeout: 7000 });
const backgroundBlue = await backgroundContainer.evaluate(el => getComputedStyle(el).backgroundColor);
// Check if background is blue (string color name or contains blue)
expect(backgroundBlue.toLowerCase()).toMatch(/blue|rgb\(.*,.*,.*\)|#[0-9a-f]{6}/);

Expand All @@ -62,15 +63,10 @@ test("[MastraAgentLocal] Agentic Chat changes background on message and reset",
);
await waitForAIResponse(page);

const backgroundPink = await chat.getBackground();
expect(backgroundPink).not.toBe(backgroundBlue);
await expect(backgroundContainer).not.toHaveCSS('background-color', backgroundBlue, { timeout: 7000 });
const backgroundPink = await backgroundContainer.evaluate(el => getComputedStyle(el).backgroundColor);
// Check if background is pink (string color name or contains pink)
expect(backgroundPink.toLowerCase()).toMatch(/pink|rgb\(.*,.*,.*\)|#[0-9a-f]{6}/);

// 3. Reset to default
await chat.sendMessage("Reset the background color to default");
await chat.assertUserMessageVisible("Reset the background color to default");
await waitForAIResponse(page);
});
});

Expand Down
16 changes: 6 additions & 10 deletions typescript-sdk/apps/dojo/e2e/tests/pydanticAITests/agenticChatPage.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@ test("[PydanticAI] Agentic Chat changes background on message and reset", async
await chat.agentGreeting.waitFor({ state: "visible" });

// Store initial background color
const initialBackground = await chat.getBackground();
const backgroundContainer = page.locator('[data-testid="background-container"]')
const initialBackground = await backgroundContainer.evaluate(el => getComputedStyle(el).backgroundColor);
console.log("Initial background color:", initialBackground);

// 1. Send message to change background to blue
Expand All @@ -50,8 +51,8 @@ test("[PydanticAI] Agentic Chat changes background on message and reset", async
);
await waitForAIResponse(page);

const backgroundBlue = await chat.getBackground();
expect(backgroundBlue).not.toBe(initialBackground);
await expect(backgroundContainer).not.toHaveCSS('background-color', initialBackground, { timeout: 7000 });
const backgroundBlue = await backgroundContainer.evaluate(el => getComputedStyle(el).backgroundColor);
// Check if background is blue (string color name or contains blue)
expect(backgroundBlue.toLowerCase()).toMatch(/blue|rgb\(.*,.*,.*\)|#[0-9a-f]{6}/);

Expand All @@ -62,15 +63,10 @@ test("[PydanticAI] Agentic Chat changes background on message and reset", async
);
await waitForAIResponse(page);

const backgroundPink = await chat.getBackground();
expect(backgroundPink).not.toBe(backgroundBlue);
await expect(backgroundContainer).not.toHaveCSS('background-color', backgroundBlue, { timeout: 7000 });
const backgroundPink = await backgroundContainer.evaluate(el => getComputedStyle(el).backgroundColor);
// Check if background is pink (string color name or contains pink)
expect(backgroundPink.toLowerCase()).toMatch(/pink|rgb\(.*,.*,.*\)|#[0-9a-f]{6}/);

// 3. Reset to default
await chat.sendMessage("Reset the background color");
await chat.assertUserMessageVisible("Reset the background color");
await waitForAIResponse(page);
});
});

Expand Down
16 changes: 6 additions & 10 deletions typescript-sdk/apps/dojo/e2e/tests/vercelAISdkTests/agenticChatPage.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@ test("[Vercel AI SDK] Agentic Chat changes background on message and reset", asy
await chat.agentGreeting.waitFor({ state: "visible" });

// Store initial background color
const initialBackground = await chat.getBackground();
const backgroundContainer = page.locator('[data-testid="background-container"]')
const initialBackground = await backgroundContainer.evaluate(el => getComputedStyle(el).backgroundColor);
console.log("Initial background color:", initialBackground);

// 1. Send message to change background to blue
Expand All @@ -50,8 +51,8 @@ test("[Vercel AI SDK] Agentic Chat changes background on message and reset", asy
);
await waitForAIResponse(page);

const backgroundBlue = await chat.getBackground();
expect(backgroundBlue).not.toBe(initialBackground);
await expect(backgroundContainer).not.toHaveCSS('background-color', initialBackground, { timeout: 7000 });
const backgroundBlue = await backgroundContainer.evaluate(el => getComputedStyle(el).backgroundColor);
// Check if background is blue (string color name or contains blue)
expect(backgroundBlue.toLowerCase()).toMatch(/blue|rgb\(.*,.*,.*\)|#[0-9a-f]{6}/);

Expand All @@ -62,15 +63,10 @@ test("[Vercel AI SDK] Agentic Chat changes background on message and reset", asy
);
await waitForAIResponse(page);

const backgroundPink = await chat.getBackground();
expect(backgroundPink).not.toBe(backgroundBlue);
await expect(backgroundContainer).not.toHaveCSS('background-color', backgroundBlue, { timeout: 7000 });
const backgroundPink = await backgroundContainer.evaluate(el => getComputedStyle(el).backgroundColor);
// Check if background is pink (string color name or contains pink)
expect(backgroundPink.toLowerCase()).toMatch(/pink|rgb\(.*,.*,.*\)|#[0-9a-f]{6}/);

// 3. Reset to default
await chat.sendMessage("Reset the background color to default");
await chat.assertUserMessageVisible("Reset the background color to default");
await waitForAIResponse(page);
});
});

Expand Down
2 changes: 1 addition & 1 deletion typescript-sdk/apps/dojo/src/app/[integrationId]/feature/agentic_chat/page.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ const Chat = () => {
});

return (
<div className="flex justify-center items-center h-full w-full" style={{ background }}>
<div className="flex justify-center items-center h-full w-full" data-testid="background-container" style={{ background }}>
<div className="h-full w-full md:w-8/10 md:h-8/10 rounded-lg">
<CopilotChat
className="h-full rounded-2xl"
Expand Down
Loading
Loading