Skip to content

fix(e2e): improve toolBasedGenUI tests for consistency #469

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

fix(e2e): improve toolBasedGenUI tests for consistency #469

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 33 additions & 13 deletions typescript-sdk/apps/dojo/e2e/featurePages/ToolBaseGenUIPage.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { Page, Locator, expect } from '@playwright/test';
import { Page, Locator, expect } from "@playwright/test";

export class ToolBaseGenUIPage {
readonly page: Page;
Expand All @@ -13,10 +13,10 @@ export class ToolBaseGenUIPage {
constructor(page: Page) {
this.page = page;
this.haikuAgentIntro = page.getByText("I'm a haiku generator 👋. How can I help you?");
this.messageBox = page.getByRole('textbox', { name: 'Type a message...' });
this.messageBox = page.getByRole("textbox", { name: "Type a message..." });
this.sendButton = page.locator('[data-test-id="copilot-chat-ready"]');
this.haikuBlock = page.locator('[data-testid="haiku-card"]');
this.applyButton = page.getByRole('button', { name: 'Apply' });
this.applyButton = page.getByRole("button", { name: "Apply" });
this.japaneseLines = page.locator('[data-testid="haiku-line"]');
}

Expand All @@ -29,12 +29,15 @@ export class ToolBaseGenUIPage {
async checkGeneratedHaiku() {
await this.page.locator('[data-testid="haiku-card"]').last().isVisible();
const mostRecentCard = this.page.locator('[data-testid="haiku-card"]').last();
await mostRecentCard.locator('[data-testid="haiku-line"]').first().waitFor({ state: 'visible', timeout: 10000 });
await mostRecentCard
.locator('[data-testid="haiku-line"]')
.first()
.waitFor({ state: "visible", timeout: 10000 });
}

async extractChatHaikuContent(page: Page): Promise<string> {
await page.waitForTimeout(3000);
await page.locator('[data-testid="haiku-card"]').first().waitFor({ state: 'visible' });
await page.locator('[data-testid="haiku-card"]').first().waitFor({ state: "visible" });
const allHaikuCards = page.locator('[data-testid="haiku-card"]');
const cardCount = await allHaikuCards.count();
let chatHaikuContainer;
Expand All @@ -47,7 +50,7 @@ export class ToolBaseGenUIPage {

if (linesCount > 0) {
try {
await chatHaikuLines.first().waitFor({ state: 'visible', timeout: 5000 });
await chatHaikuLines.first().waitFor({ state: "visible", timeout: 5000 });
break;
} catch (error) {
continue;
Expand All @@ -56,19 +59,19 @@ export class ToolBaseGenUIPage {
}

if (!chatHaikuLines) {
throw new Error('No haiku cards with visible lines found');
throw new Error("No haiku cards with visible lines found");
}

const count = await chatHaikuLines.count();
const lines: string[] = [];

for (let i = 0; i < count; i++) {
const haikuLine = chatHaikuLines.nth(i);
const japaneseText = await haikuLine.locator('p').first().innerText();
const japaneseText = await haikuLine.locator("p").first().innerText();
lines.push(japaneseText);
}

const chatHaikuContent = lines.join('').replace(/\s/g, '');
const chatHaikuContent = lines.join("").replace(/\s/g, "");
return chatHaikuContent;
}

Expand All @@ -80,12 +83,12 @@ export class ToolBaseGenUIPage {
if (mainCount > 0) {
for (let i = 0; i < mainCount; i++) {
const haikuLine = mainDisplayLines.nth(i);
const japaneseText = await haikuLine.locator('p').first().innerText();
const japaneseText = await haikuLine.locator("p").first().innerText();
lines.push(japaneseText);
}
}

const mainHaikuContent = lines.join('').replace(/\s/g, '');
const mainHaikuContent = lines.join("").replace(/\s/g, "");
return mainHaikuContent;
}

Expand All @@ -96,7 +99,7 @@ export class ToolBaseGenUIPage {

const mainHaikuContent = await this.extractMainDisplayHaikuContent(page);

if (mainHaikuContent === '') {
if (mainHaikuContent === "") {
expect(chatHaikuContent.length).toBeGreaterThan(0);
return;
}
Expand All @@ -111,4 +114,21 @@ export class ToolBaseGenUIPage {
expect(updatedMainContent).toBe(chatHaikuContent);
}
}
}

async waitForMainDisplayHaiku(page: Page, expectedContent: string): Promise<void> {
await page.waitForFunction(
(expectedContent) => {
const mainLines = Array.from(
document.querySelectorAll('[data-testid="main-haiku-line"] p:first-child'),
);
const mainContent = mainLines
.map((el) => el.textContent)
.join("")
.replace(/\s/g, "");
return mainContent === expectedContent;
},
expectedContent,
{ timeout: 10000 },
);
}
}
20 changes: 12 additions & 8 deletions typescript-sdk/apps/dojo/e2e/tests/adkMiddlewareTests/toolBasedGenUIPage.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,7 @@
const pageURL = "/adk-middleware/feature/tool_based_generative_ui";

test.describe("Tool Based Generative UI Feature", () => {
test('[ADK Middleware] Haiku generation and display verification', async ({
page,
}) => {
test("[ADK Middleware] Haiku generation and display verification", async ({ page }) => {
await page.goto(pageURL);

const genAIAgent = new ToolBaseGenUIPage(page);
Expand All @@ -17,7 +15,7 @@
await genAIAgent.checkHaikuDisplay(page);
});

test('[ADK Middleware] Haiku generation and UI consistency for two different prompts', async ({
test("[ADK Middleware] Haiku generation and UI consistency for two different prompts", async ({
page,
}) => {
await page.goto(pageURL);
Expand All @@ -29,11 +27,17 @@
const prompt1 = 'Generate Haiku for "I will always win"';
await genAIAgent.generateHaiku(prompt1);
await genAIAgent.checkGeneratedHaiku();
await genAIAgent.checkHaikuDisplay(page);
const haiku1Content = await genAIAgent.extractChatHaikuContent(page);
await genAIAgent.waitForMainDisplayHaiku(page, haiku1Content);

const prompt2 = 'Generate Haiku for "The moon shines bright"';
await genAIAgent.generateHaiku(prompt2);
await genAIAgent.checkGeneratedHaiku(); // Wait for second haiku to be generated
await genAIAgent.checkHaikuDisplay(page); // Now compare the second haiku
await genAIAgent.checkGeneratedHaiku();
const haiku2Content = await genAIAgent.extractChatHaikuContent(page);

// Verify haikus are different
expect(haiku1Content).not.toBe(haiku2Content);

Check failure on line 39 in typescript-sdk/apps/dojo/e2e/tests/adkMiddlewareTests/toolBasedGenUIPage.spec.ts

View workflow job for this annotation

GitHub Actions / adk-middleware

[chromium] › tests/adkMiddlewareTests/toolBasedGenUIPage.spec.ts:18:7 › Tool Based Generative UI Feature › [ADK Middleware] Haiku generation and UI consistency for two different prompts 

  1) [chromium] › tests/adkMiddlewareTests/toolBasedGenUIPage.spec.ts:18:7 › Tool Based Generative UI Feature › [ADK Middleware] Haiku generation and UI consistency for two different prompts 

    Retry #2 ───────────────────────────────────────────────────────────────────────────────────────
    Error: expect(received).not.toBe(expected) // Object.is equality

    Expected: not "勝つという心はいつも輝き放つ"

      37 |
      38 |     // Verify haikus are different
    > 39 |     expect(haiku1Content).not.toBe(haiku2Content);
         |                               ^
      40 |
      41 |     await genAIAgent.waitForMainDisplayHaiku(page, haiku2Content);
      42 |   });
        at /home/runner/work/ag-ui/ag-ui/typescript-sdk/apps/dojo/e2e/tests/adkMiddlewareTests/toolBasedGenUIPage.spec.ts:39:31

Check failure on line 39 in typescript-sdk/apps/dojo/e2e/tests/adkMiddlewareTests/toolBasedGenUIPage.spec.ts

View workflow job for this annotation

GitHub Actions / adk-middleware

[chromium] › tests/adkMiddlewareTests/toolBasedGenUIPage.spec.ts:18:7 › Tool Based Generative UI Feature › [ADK Middleware] Haiku generation and UI consistency for two different prompts 

  1) [chromium] › tests/adkMiddlewareTests/toolBasedGenUIPage.spec.ts:18:7 › Tool Based Generative UI Feature › [ADK Middleware] Haiku generation and UI consistency for two different prompts 

    Retry #1 ───────────────────────────────────────────────────────────────────────────────────────
    Error: expect(received).not.toBe(expected) // Object.is equality

    Expected: not "勝ち続け道は開ける光の中"

      37 |
      38 |     // Verify haikus are different
    > 39 |     expect(haiku1Content).not.toBe(haiku2Content);
         |                               ^
      40 |
      41 |     await genAIAgent.waitForMainDisplayHaiku(page, haiku2Content);
      42 |   });
        at /home/runner/work/ag-ui/ag-ui/typescript-sdk/apps/dojo/e2e/tests/adkMiddlewareTests/toolBasedGenUIPage.spec.ts:39:31

Check failure on line 39 in typescript-sdk/apps/dojo/e2e/tests/adkMiddlewareTests/toolBasedGenUIPage.spec.ts

View workflow job for this annotation

GitHub Actions / adk-middleware

[chromium] › tests/adkMiddlewareTests/toolBasedGenUIPage.spec.ts:18:7 › Tool Based Generative UI Feature › [ADK Middleware] Haiku generation and UI consistency for two different prompts 

  1) [chromium] › tests/adkMiddlewareTests/toolBasedGenUIPage.spec.ts:18:7 › Tool Based Generative UI Feature › [ADK Middleware] Haiku generation and UI consistency for two different prompts 
    Error: expect(received).not.toBe(expected) // Object.is equality

    Expected: not "我は常に勝つ道を切り開き栄光を掴む"

      37 |
      38 |     // Verify haikus are different
    > 39 |     expect(haiku1Content).not.toBe(haiku2Content);
         |                               ^
      40 |
      41 |     await genAIAgent.waitForMainDisplayHaiku(page, haiku2Content);
      42 |   });
        at /home/runner/work/ag-ui/ag-ui/typescript-sdk/apps/dojo/e2e/tests/adkMiddlewareTests/toolBasedGenUIPage.spec.ts:39:31

await genAIAgent.waitForMainDisplayHaiku(page, haiku2Content);
});
});
});
25 changes: 13 additions & 12 deletions typescript-sdk/apps/dojo/e2e/tests/agnoTests/toolBasedGenUIPage.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,9 @@
import { test, expect } from "@playwright/test";
import { ToolBaseGenUIPage } from "../../featurePages/ToolBaseGenUIPage";

const pageURL =
"/agno/feature/tool_based_generative_ui";
const pageURL = "/agno/feature/tool_based_generative_ui";

test('[Agno] Haiku generation and display verification', async ({
page,
}) => {
test("[Agno] Haiku generation and display verification", async ({ page }) => {
await page.goto(pageURL);

const genAIAgent = new ToolBaseGenUIPage(page);
Expand All @@ -17,9 +14,7 @@ test('[Agno] Haiku generation and display verification', async ({
await genAIAgent.checkHaikuDisplay(page);
});

test('[Agno] Haiku generation and UI consistency for two different prompts', async ({
page,
}) => {
test("[Agno] Haiku generation and UI consistency for two different prompts", async ({ page }) => {
await page.goto(pageURL);

const genAIAgent = new ToolBaseGenUIPage(page);
Expand All @@ -29,10 +24,16 @@ test('[Agno] Haiku generation and UI consistency for two different prompts', asy
const prompt1 = 'Generate Haiku for "I will always win"';
await genAIAgent.generateHaiku(prompt1);
await genAIAgent.checkGeneratedHaiku();
await genAIAgent.checkHaikuDisplay(page);
const haiku1Content = await genAIAgent.extractChatHaikuContent(page);
await genAIAgent.waitForMainDisplayHaiku(page, haiku1Content);

const prompt2 = 'Generate Haiku for "The moon shines bright"';
await genAIAgent.generateHaiku(prompt2);
await genAIAgent.checkGeneratedHaiku(); // Wait for second haiku to be generated
await genAIAgent.checkHaikuDisplay(page); // Now compare the second haiku
});
await genAIAgent.checkGeneratedHaiku();
const haiku2Content = await genAIAgent.extractChatHaikuContent(page);

// Verify haikus are different
expect(haiku1Content).not.toBe(haiku2Content);

await genAIAgent.waitForMainDisplayHaiku(page, haiku2Content);
});
25 changes: 13 additions & 12 deletions typescript-sdk/apps/dojo/e2e/tests/crewAITests/toolBasedGenUIPage.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,9 @@
import { test, expect } from "@playwright/test";
import { ToolBaseGenUIPage } from "../../featurePages/ToolBaseGenUIPage";

const pageURL =
"/crewai/feature/tool_based_generative_ui";
const pageURL = "/crewai/feature/tool_based_generative_ui";

test('[CrewAI] Haiku generation and display verification', async ({
page,
}) => {
test("[CrewAI] Haiku generation and display verification", async ({ page }) => {
await page.goto(pageURL);

const genAIAgent = new ToolBaseGenUIPage(page);
Expand All @@ -17,9 +14,7 @@ test('[CrewAI] Haiku generation and display verification', async ({
await genAIAgent.checkHaikuDisplay(page);
});

test('[CrewAI] Haiku generation and UI consistency for two different prompts', async ({
page,
}) => {
test("[CrewAI] Haiku generation and UI consistency for two different prompts", async ({ page }) => {
await page.goto(pageURL);

const genAIAgent = new ToolBaseGenUIPage(page);
Expand All @@ -29,10 +24,16 @@ test('[CrewAI] Haiku generation and UI consistency for two different prompts', a
const prompt1 = 'Generate Haiku for "I will always win"';
await genAIAgent.generateHaiku(prompt1);
await genAIAgent.checkGeneratedHaiku();
await genAIAgent.checkHaikuDisplay(page);
const haiku1Content = await genAIAgent.extractChatHaikuContent(page);
await genAIAgent.waitForMainDisplayHaiku(page, haiku1Content);

const prompt2 = 'Generate Haiku for "The moon shines bright"';
await genAIAgent.generateHaiku(prompt2);
await genAIAgent.checkGeneratedHaiku(); // Wait for second haiku to be generated
await genAIAgent.checkHaikuDisplay(page); // Now compare the second haiku
});
await genAIAgent.checkGeneratedHaiku();
const haiku2Content = await genAIAgent.extractChatHaikuContent(page);

// Verify haikus are different
expect(haiku1Content).not.toBe(haiku2Content);

await genAIAgent.waitForMainDisplayHaiku(page, haiku2Content);
});
21 changes: 12 additions & 9 deletions typescript-sdk/apps/dojo/e2e/tests/langgraphFastAPITests/toolBasedGenUIPage.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,9 @@
import { test, expect } from "@playwright/test";
import { ToolBaseGenUIPage } from "../../featurePages/ToolBaseGenUIPage";

const pageURL =
"/langgraph-fastapi/feature/tool_based_generative_ui";
const pageURL = "/langgraph-fastapi/feature/tool_based_generative_ui";

test('[LangGraph FastAPI] Haiku generation and display verification', async ({
page,
}) => {
test("[LangGraph FastAPI] Haiku generation and display verification", async ({ page }) => {
await page.goto(pageURL);

const genAIAgent = new ToolBaseGenUIPage(page);
Expand All @@ -17,7 +14,7 @@ test('[LangGraph FastAPI] Haiku generation and display verification', async ({
await genAIAgent.checkHaikuDisplay(page);
});

test('[LangGraph FastAPI] Haiku generation and UI consistency for two different prompts', async ({
test("[LangGraph FastAPI] Haiku generation and UI consistency for two different prompts", async ({
page,
}) => {
await page.goto(pageURL);
Expand All @@ -29,10 +26,16 @@ test('[LangGraph FastAPI] Haiku generation and UI consistency for two different
const prompt1 = 'Generate Haiku for "I will always win"';
await genAIAgent.generateHaiku(prompt1);
await genAIAgent.checkGeneratedHaiku();
await genAIAgent.checkHaikuDisplay(page);
const haiku1Content = await genAIAgent.extractChatHaikuContent(page);
await genAIAgent.waitForMainDisplayHaiku(page, haiku1Content);

const prompt2 = 'Generate Haiku for "The moon shines bright"';
await genAIAgent.generateHaiku(prompt2);
await genAIAgent.checkGeneratedHaiku();
await genAIAgent.checkHaikuDisplay(page);
});
const haiku2Content = await genAIAgent.extractChatHaikuContent(page);

// Verify haikus are different
expect(haiku1Content).not.toBe(haiku2Content);

await genAIAgent.waitForMainDisplayHaiku(page, haiku2Content);
});
21 changes: 12 additions & 9 deletions typescript-sdk/apps/dojo/e2e/tests/langgraphPythonTests/toolBasedGenUIPage.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,9 @@
import { test, expect } from "@playwright/test";
import { ToolBaseGenUIPage } from "../../featurePages/ToolBaseGenUIPage";

const pageURL =
"/langgraph/feature/tool_based_generative_ui";
const pageURL = "/langgraph/feature/tool_based_generative_ui";

test('[LangGraph] Haiku generation and display verification', async ({
page,
}) => {
test("[LangGraph] Haiku generation and display verification", async ({ page }) => {
await page.goto(pageURL);

const genAIAgent = new ToolBaseGenUIPage(page);
Expand All @@ -17,7 +14,7 @@ test('[LangGraph] Haiku generation and display verification', async ({
await genAIAgent.checkHaikuDisplay(page);
});

test('[LangGraph] Haiku generation and UI consistency for two different prompts', async ({
test("[LangGraph] Haiku generation and UI consistency for two different prompts", async ({
page,
}) => {
await page.goto(pageURL);
Expand All @@ -29,10 +26,16 @@ test('[LangGraph] Haiku generation and UI consistency for two different prompts'
const prompt1 = 'Generate Haiku for "I will always win"';
await genAIAgent.generateHaiku(prompt1);
await genAIAgent.checkGeneratedHaiku();
await genAIAgent.checkHaikuDisplay(page);
const haiku1Content = await genAIAgent.extractChatHaikuContent(page);
await genAIAgent.waitForMainDisplayHaiku(page, haiku1Content);

const prompt2 = 'Generate Haiku for "The moon shines bright"';
await genAIAgent.generateHaiku(prompt2);
await genAIAgent.checkGeneratedHaiku();
await genAIAgent.checkHaikuDisplay(page);
});
const haiku2Content = await genAIAgent.extractChatHaikuContent(page);

// Verify haikus are different
expect(haiku1Content).not.toBe(haiku2Content);

await genAIAgent.waitForMainDisplayHaiku(page, haiku2Content);
});
21 changes: 12 additions & 9 deletions typescript-sdk/apps/dojo/e2e/tests/langgraphTypescriptTests/toolBasedGenUIPage.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,9 @@
import { test, expect } from "@playwright/test";
import { ToolBaseGenUIPage } from "../../featurePages/ToolBaseGenUIPage";

const pageURL =
"/langgraph-typescript/feature/tool_based_generative_ui";
const pageURL = "/langgraph-typescript/feature/tool_based_generative_ui";

test('[LangGraph] Haiku generation and display verification', async ({
page,
}) => {
test("[LangGraph] Haiku generation and display verification", async ({ page }) => {
await page.goto(pageURL);

const genAIAgent = new ToolBaseGenUIPage(page);
Expand All @@ -17,7 +14,7 @@ test('[LangGraph] Haiku generation and display verification', async ({
await genAIAgent.checkHaikuDisplay(page);
});

test('[LangGraph] Haiku generation and UI consistency for two different prompts', async ({
test("[LangGraph] Haiku generation and UI consistency for two different prompts", async ({
page,
}) => {
await page.goto(pageURL);
Expand All @@ -29,10 +26,16 @@ test('[LangGraph] Haiku generation and UI consistency for two different prompts'
const prompt1 = 'Generate Haiku for "I will always win"';
await genAIAgent.generateHaiku(prompt1);
await genAIAgent.checkGeneratedHaiku();
await genAIAgent.checkHaikuDisplay(page);
const haiku1Content = await genAIAgent.extractChatHaikuContent(page);
await genAIAgent.waitForMainDisplayHaiku(page, haiku1Content);

const prompt2 = 'Generate Haiku for "The moon shines bright"';
await genAIAgent.generateHaiku(prompt2);
await genAIAgent.checkGeneratedHaiku();
await genAIAgent.checkHaikuDisplay(page);
});
const haiku2Content = await genAIAgent.extractChatHaikuContent(page);

// Verify haikus are different
expect(haiku1Content).not.toBe(haiku2Content);

await genAIAgent.waitForMainDisplayHaiku(page, haiku2Content);
});
Loading
Loading