test: Implemented Log execution time per prompt in Report for KM generic

Author: Avijit-Microsoft

tests/e2e-test/README.md

@@ -24,7 +24,7 @@ Installing Playwright Pytest from Virtual Environment
 
 Run test cases
 
-- To run test cases from your 'tests' folder : "pytest --html=report.html --self-contained-html"
+- To run test cases from your 'tests/e2e-test' folder : "pytest --html=report.html --self-contained-html"
 
 Create .env file in project root level with web app url and client credentials
 

tests/e2e-test/pages/HomePage.py

@@ -6,9 +6,13 @@ class HomePage(BasePage):
     SEND_BUTTON = "//button[@title='Send Question']"
     SHOW_CHAT_HISTORY_BUTTON = "//button[normalize-space()='Show Chat History']"
     HIDE_CHAT_HISTORY_BUTTON = "//button[normalize-space()='Hide Chat History']"
-    CHAT_HISTORY_NAME = "//div[contains(translate(text(), 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz'), 'call')]"
+    CHAT_HISTORY_NAME = "//div[contains(@class, 'ChatHistoryListItemCell_chatTitle')]"
     CLEAR_CHAT_HISTORY_MENU = "//button[@id='moreButton']"
     CLEAR_CHAT_HISTORY = "//button[@role='menuitem']"
+    REFERENCE_LINKS_IN_RESPONSE = "//span[@role='button' and contains(@class, 'citationContainer')]"
+    CLOSE_BUTTON = "svg[role='button'][tabindex='0']"
+
+
 
     def __init__(self, page):
         self.page = page
@@ -40,7 +44,7 @@ def show_chat_history(self):
         self.page.wait_for_load_state('networkidle')
         self.page.wait_for_timeout(2000)
         try:
-            expect(self.page.locator(self.CHAT_HISTORY_NAME)).to_be_visible(timeout=7000)
+            expect(self.page.locator(self.CHAT_HISTORY_NAME)).to_be_visible(timeout=9000)
         except AssertionError:
             raise AssertionError("Chat history name was not visible on the page within the expected time.")
 
@@ -67,5 +71,34 @@ def close_chat_history(self):
         self.page.locator(self.HIDE_CHAT_HISTORY_BUTTON).click()
         self.page.wait_for_load_state('networkidle')
         self.page.wait_for_timeout(2000)
+    
+    def click_reference_link_in_response(self):
+        # Click on reference link response
+        BasePage.scroll_into_view(self, self.page.locator(self.REFERENCE_LINKS_IN_RESPONSE))
+        self.page.wait_for_timeout(2000)
+        reference_links = self.page.locator(self.REFERENCE_LINKS_IN_RESPONSE)
+        reference_links.nth(reference_links.count() - 1).click()
+        # self.page.locator(self.REFERENCE_LINKS_IN_RESPONSE).click()
+        self.page.wait_for_load_state('networkidle')
+        self.page.wait_for_timeout(2000)
+ 
+ 
+    def close_citation(self):
+        self.page.wait_for_timeout(3000)
+        
+        close_btn = self.page.locator(self.CLOSE_BUTTON)
+        close_btn.wait_for(state="attached", timeout=5000)
+        # bring it into view just in case
+        close_btn.scroll_into_view_if_needed()
+        # force the click, bypassing the aria-hidden check
+        close_btn.click(force=True)
+        self.page.wait_for_timeout(5000)
+
+    def has_reference_link(self):
+        # Get all assistant messages
+        assistant_messages = self.page.locator("div.chat-message.assistant")
+        last_assistant = assistant_messages.nth(assistant_messages.count() - 1)
 
-  
+        # Use XPath properly by prefixing with 'xpath='
+        reference_links = last_assistant.locator("xpath=.//span[@role='button' and contains(@class, 'citationContainer')]")
+        return reference_links.count() > 0

tests/e2e-test/pytest.ini

@@ -3,4 +3,4 @@ log_cli = true
 log_cli_level = INFO
 log_file = logs/tests.log
 log_file_level = INFO
-addopts = -p no:warnings
+addopts = -p no:warnings --tb=short

tests/e2e-test/requirements.txt

@@ -4,3 +4,4 @@ python-dotenv
 pytest-check
 pytest-html
 py
+beautifulsoup4

tests/e2e-test/sample_dotenv_file.txt

@@ -1,2 +1,2 @@
 url = 'web app url'
-api_url = 'api app url"
+api_url = "api app url"

tests/e2e-test/testdata/prompts.json

@@ -2,6 +2,7 @@
     "questions":[
 
 "Total number of calls by date for last 7 days",
+"Generate a line chart",
 "Show average handling time by topics in minutes",
 "What are top 7 challenges user reported?",
 "Give a summary of billing issues",

tests/e2e-test/tests/conftest.py

@@ -7,7 +7,10 @@
 from dotenv import load_dotenv
 import os
 from py.xml import html # type: ignore
-
+import io
+import logging
+from bs4 import BeautifulSoup
+import atexit
 
 @pytest.fixture(scope="session")
 def login_logout():
@@ -28,25 +31,74 @@ def login_logout():
         # perform close the browser
         browser.close()
 
+log_streams = {}
 
 @pytest.hookimpl(tryfirst=True)
-def pytest_html_report_title(report):
-    report.title = "Automation_KM_Generic"
+def pytest_runtest_setup(item):
+    # Prepare StringIO for capturing logs
+    stream = io.StringIO()
+    handler = logging.StreamHandler(stream)
+    handler.setLevel(logging.INFO)
 
+    logger = logging.getLogger()
+    logger.addHandler(handler)
 
-# Add a column for descriptions
-def pytest_html_results_table_header(cells):
-    cells.insert(1, html.th("Description"))
+    # Save handler and stream
+    log_streams[item.nodeid] = (handler, stream)
 
-def pytest_html_results_table_row(report, cells):
-    cells.insert(1, html.td(report.description if hasattr(report, "description") else ""))
 
-# Add logs and docstring to report
 @pytest.hookimpl(hookwrapper=True)
 def pytest_runtest_makereport(item, call):
     outcome = yield
     report = outcome.get_result()
-    report.description = str(item.function.__doc__)
-    os.makedirs("logs", exist_ok=True)
-    extra = getattr(report, 'extra', [])
-    report.extra = extra
+
+    handler, stream = log_streams.get(item.nodeid, (None, None))
+
+    if handler and stream:
+        # Make sure logs are flushed
+        handler.flush()
+        log_output = stream.getvalue()
+
+        # Only remove the handler, don't close the stream yet
+        logger = logging.getLogger()
+        logger.removeHandler(handler)
+
+        # Store the log output on the report object for HTML reporting
+        report.description = f"<pre>{log_output.strip()}</pre>"
+
+        # Clean up references
+        log_streams.pop(item.nodeid, None)
+    else:
+        report.description = ""
+
+def pytest_collection_modifyitems(items):
+    for item in items:
+        if hasattr(item, 'callspec'):
+            prompt = item.callspec.params.get("prompt")
+            if prompt:
+                item._nodeid = prompt  # This controls how the test name appears in the report
+
+def rename_duration_column():
+    report_path = os.path.abspath("report.html")  # or your report filename
+    if not os.path.exists(report_path):
+        print("Report file not found, skipping column rename.")
+        return
+
+    with open(report_path, 'r', encoding='utf-8') as f:
+        soup = BeautifulSoup(f, 'html.parser')
+
+    # Find and rename the header
+    headers = soup.select('table#results-table thead th')
+    for th in headers:
+        if th.text.strip() == 'Duration':
+            th.string = 'Execution Time'
+            #print("Renamed 'Duration' to 'Execution Time'")
+            break
+    else:
+        print("'Duration' column not found in report.")
+
+    with open(report_path, 'w', encoding='utf-8') as f:
+        f.write(str(soup))
+
+# Register this function to run after everything is done
+atexit.register(rename_duration_column)

tests/e2e-test/tests/test_km_gp_tc.py

@@ -1,39 +1,91 @@
+import time
+import logging
+import pytest
+from pytest_check import check
 from pages.HomePage import HomePage
 from config.constants import *
-from pytest_check import check
-import logging
+import io
 
 logger = logging.getLogger(__name__)
 
-def test_KM_Generic_Golden_path_test(login_logout):
+# Helper to validate the final response text
+def _validate_response_text(page, question):
+    response_text = page.locator("//p")
+    last_response = response_text.nth(response_text.count() - 1).text_content()
+    check.not_equal(
+        "I cannot answer this question from the data available. Please rephrase or add more details.",
+        last_response,
+        f"Invalid response for: {question}"
+    )
+    check.not_equal(
+        "Chart cannot be generated.",
+        last_response,
+        f"Invalid response for: {question}"
+    )
+
+# Helper to check and close citation if it exists
+# def _check_and_close_citation(home):
+#     if home.has_reference_link():
+#         logger.info("Step: Reference link found. Opening citation.")
+#         home.click_reference_link_in_response()
+#         logger.info("Step: Closing citation.")
+#         home.close_citation()
+
+# Define test steps
+test_steps = [
+    ("Validate home page is loaded", lambda home: home.home_page_load()),
+    ("Validate delete chat history", lambda home: home.delete_chat_history()),
+]
+
+# Add golden path question prompts
+for i, q in enumerate(questions, start=1):
+    def _question_step(home, q=q):  # q is default arg to avoid late binding
+        home.enter_chat_question(q)
+        home.click_send_button()
+        home.validate_response_status(q)
+        _validate_response_text(home.page, q)
+        
+        # Include citation check directly
+        if home.has_reference_link():
+            logger.info(f"[{q}] Reference link found. Opening citation.")
+            home.click_reference_link_in_response()
+            logger.info(f"[{q}] Closing citation.")
+            home.close_citation()
+
+    test_steps.append((f"Validate response for GP Prompt: {q}", _question_step))
+
+# Final chat history validation
+test_steps.extend([
+    ("Validate chat history is saved", lambda home: home.show_chat_history()),
+    ("Validate chat history is closed", lambda home: home.close_chat_history()),
+])
+
+# Test ID display for reporting
+test_ids = [f"{i+1:02d}. {desc}" for i, (desc, _) in enumerate(test_steps)]
+
+@pytest.mark.parametrize("description, step", test_steps, ids=test_ids)
+def test_KM_Generic_Golden_Path(login_logout, description, step, request):
+    request.node._nodeid = description
 
-    """Validate Golden path test case for KM Generic Accelerator """
     page = login_logout
     home_page = HomePage(page)
-    
-    logger.info("Step 1: Validate home page is loaded.")
-    home_page.home_page_load()
-
-    logger.info("Step 2: Validate delete chat history.")
-    home_page.delete_chat_history()
-
-    logger.info("Step 3: Validate GP Prompts response.")
-    for question in questions:
-        #enter question
-        home_page.enter_chat_question(question)
-        # click on send button
-        home_page.click_send_button()
-        #validate the response status
-        home_page.validate_response_status(question)
-        #validate response text
-        response_text = page.locator("//p")
-        #assert response text
-        check.not_equal("I cannot answer this question from the data available. Please rephrase or add more details.", response_text.nth(response_text.count() - 1).text_content(), f"Invalid response for : {question}")
-        check.not_equal("Chart cannot be generated.", response_text.nth(response_text.count() - 1).text_content(), f"Invalid response for : {question}")
-        
-       
-    logger.info("Step 4: Validate chat history.")
-    home_page.show_chat_history()
+    home_page.page = page
+
+    log_capture = io.StringIO()
+    handler = logging.StreamHandler(log_capture)
+    logger.addHandler(handler)
+
+    logger.info(f"Running test step: {description}")
+    start = time.time()
+
+    try:
+        step(home_page)
+    finally:
+        duration = time.time() - start
+        logger.info(f"Execution Time for '{description}': {duration:.2f}s")
+        logger.removeHandler(handler)
 
-    logger.info("Step 5: Validate close chat history.")
-    home_page.close_chat_history()
+        # Attach logs
+        request.node._report_sections.append((
+            "call", "log", log_capture.getvalue()
+        ))