test: Implement Log execution time per prompt in Report for CWYD (#1832)

Author: Rohini-Microsoft

tests/e2e-test/pages/adminPage.py

@@ -28,3 +28,8 @@ def __init__(self, page):
     def click_delete_data_tab(self):
         self.page.locator(self.DELETE_DATA_TAB).click()
         self.page.wait_for_timeout(5000)
+
+    def assert_admin_page_title(self, admin_page):
+        actual_title = self.page.locator(admin_page.ADMIN_PAGE_TITLE).text_content()
+        expected_title = admin_page.ADMIN_PAGE_TITLE
+        assert expected_title == actual_title, f"Expected title: {expected_title}, Found: {actual_title}"

tests/e2e-test/pages/webUserPage.py

@@ -1,7 +1,6 @@
+from asyncio.log import logger
 from base.base import BasePage
 from playwright.sync_api import expect
-
-
 class WebUserPage(BasePage):
     WEB_PAGE_TITLE = "//h3[text()='Azure AI']"
     TYPE_QUESTION_TEXT_AREA = "//textarea[contains(@placeholder,'Type a new question')]"
@@ -22,6 +21,9 @@ class WebUserPage(BasePage):
     TOGGLE_CITATIONS_LIST = "[data-testid='toggle-citations-list']"
     CITATIONS_CONTAINER = "[data-testid='citations-container']"
     CITATION_BLOCK = "[data-testid='citation-block']"
+    SHOW_CHAT_HISTORY_BUTTON="//span[text()='Show Chat History']"
+    HIDE_CHAT_HISTORY_BUTTON = "//span[text()='Hide Chat History']"
+    CHAT_HISTORY_ITEM = "//div[@aria-label='chat history item']"
 
     def __init__(self, page):
         self.page = page
@@ -53,24 +55,37 @@ def click_clear_chat_icon(self):
             self.page.locator(self.CLEAR_CHAT_ICON).click()
 
     def show_chat_history(self):
-        self.page.locator(self.SHOW_CHAT_HISTORY).click()
-        self.page.wait_for_load_state("networkidle")
-        self.page.wait_for_timeout(2000)
-        expect(self.page.locator(self.CHAT_HISTORY_NAME)).to_be_visible()
+        """Click to show chat history if the button is visible."""
+        show_button = self.page.locator(self.SHOW_CHAT_HISTORY_BUTTON)
+        if show_button.is_visible():
+            show_button.click()
+            self.page.wait_for_timeout(2000)
+            expect(self.page.locator(self.CHAT_HISTORY_ITEM)).to_be_visible()
+        else:
+            logger.info("'Show' button not visible — chat history may already be shown.")
+
+    # def show_chat_history(self):
+    #     self.page.wait_for_selector(self.SHOW_CHAT_HISTORY_BUTTON)
+    #     self.page.locator(self.SHOW_CHAT_HISTORY_BUTTON).click()
+    #     self.page.wait_for_timeout(1000)
 
     def close_chat_history(self):
-        self.page.locator(self.CHAT_CLOSE_ICON).click()
-        self.page.wait_for_load_state("networkidle")
-        self.page.wait_for_timeout(2000)
+        """Click to close chat history if visible."""
+        hide_button = self.page.locator(self.HIDE_CHAT_HISTORY_BUTTON)
+        if hide_button.is_visible():
+            hide_button.click()
+            self.page.wait_for_timeout(2000)
+        else:
+            logger.info("Hide button not visible. Chat history might already be closed.")
 
     def delete_chat_history(self):
         self.page.locator(self.SHOW_CHAT_HISTORY).click()
         self.page.wait_for_timeout(2000)
         chat_history = self.page.locator("//span[contains(text(),'No chat history.')]")
         if chat_history.is_visible():
             self.page.wait_for_load_state("networkidle")
-            self.page.wait_for_timeout(2000)
-            self.page.get_by_label("hide button").click()
+            self.page.locator("button[title='Hide']").wait_for(state="visible", timeout=5000)
+            self.page.locator("button[title='Hide']").click()
 
         else:
             self.page.locator(self.CHAT_HISTORY_OPTIONS).click()

tests/e2e-test/pytest.ini

@@ -3,4 +3,4 @@ log_cli = true
 log_cli_level = INFO
 log_file = logs/tests.log
 log_file_level = INFO
-addopts = -p no:warnings
+addopts = -p no:warnings --tb=short

tests/e2e-test/requirements.txt

@@ -3,4 +3,5 @@ pytest-reporter-html1
 python-dotenv
 pytest-check
 pytest-html
-py
+py
+beautifulsoup4

tests/e2e-test/tests/conftest.py

@@ -1,54 +1,92 @@
-import os
-
 import pytest
-from config.constants import *
+import os
+import io
+import logging
+import atexit
+from bs4 import BeautifulSoup
 from playwright.sync_api import sync_playwright
-from py.xml import html  # type: ignore
+from config.constants import *
 
+log_streams = {}
 
+# ---------- FIXTURE: Login and Logout Setup ----------
 @pytest.fixture(scope="session")
 def login_logout():
-    # perform login and browser close once in a session
     with sync_playwright() as p:
         browser = p.chromium.launch(headless=False, args=["--start-maximized"])
         context = browser.new_context(no_viewport=True)
         context.set_default_timeout(80000)
         page = context.new_page()
-        # Navigate to the login URL
+
+        # Load URL and wait
         page.goto(WEB_URL)
-        # Wait for the login form to appear
         page.wait_for_load_state("networkidle")
         page.wait_for_timeout(5000)
-        # login to web url with username and password
-        # login_page = LoginPage(page)
+
+        # Uncomment if authentication is needed
         # load_dotenv()
+        # login_page = LoginPage(page)
         # login_page.authenticate(os.getenv('user_name'), os.getenv('pass_word'))
+
         yield page
         browser.close()
 
-
+# ---------- HTML Report Title ----------
 @pytest.hookimpl(tryfirst=True)
 def pytest_html_report_title(report):
     report.title = "Test_Automation_Chat_with_your_Data"
 
+# ---------- Logging Setup per Test ----------
+@pytest.hookimpl(tryfirst=True)
+def pytest_runtest_setup(item):
+    stream = io.StringIO()
+    handler = logging.StreamHandler(stream)
+    handler.setLevel(logging.INFO)
+    logger = logging.getLogger()
+    logger.addHandler(handler)
+    log_streams[item.nodeid] = (handler, stream)
 
-# Add a column for descriptions
-def pytest_html_results_table_header(cells):
-    cells.insert(1, html.th("Description"))
-
-
-def pytest_html_results_table_row(report, cells):
-    cells.insert(
-        1, html.td(report.description if hasattr(report, "description") else "")
-    )
-
-
-# Add logs and docstring to report
+# ---------- Attach Logs to HTML Report ----------
 @pytest.hookimpl(hookwrapper=True)
 def pytest_runtest_makereport(item, call):
     outcome = yield
     report = outcome.get_result()
-    report.description = str(item.function.__doc__)
-    os.makedirs("logs", exist_ok=True)
-    extra = getattr(report, "extra", [])
-    report.extra = extra
+
+    if report.when == "call":
+        question_logs = getattr(item, "_question_logs", None)
+        if question_logs:
+            for i, (question, logs) in enumerate(question_logs.items(), start=1):
+                report.sections.append((f"Q{i:02d}: {question}", logs))
+        else:
+            log = getattr(item, "_captured_log", None)
+            if log:
+                report.sections.append(("Captured Log", log))
+
+# ---------- Optional: Clean Up Node IDs for Parametrized Prompts ----------
+def pytest_collection_modifyitems(items):
+    for item in items:
+        if hasattr(item, 'callspec') and "prompt" in item.callspec.params:
+            item._nodeid = item.callspec.params["prompt"]
+
+# ---------- Rename Duration Column in HTML Report ----------
+def rename_duration_column():
+    report_path = os.path.abspath("report.html")
+    if not os.path.exists(report_path):
+        print("Report file not found, skipping column rename.")
+        return
+
+    with open(report_path, 'r', encoding='utf-8') as f:
+        soup = BeautifulSoup(f, 'html.parser')
+
+    headers = soup.select('table#results-table thead th')
+    for th in headers:
+        if th.text.strip() == 'Duration':
+            th.string = 'Execution Time'
+            break
+    else:
+        print("'Duration' column not found in report.")
+
+    with open(report_path, 'w', encoding='utf-8') as f:
+        f.write(str(soup))
+
+atexit.register(rename_duration_column)