test: Doc Gen-Create individual test case for each prompt and capture execution time for each test case

Author: Ritesh-Microsoft

tests/e2e-test/README.md

@@ -20,11 +20,11 @@ This will create a virtual environment directory named microsoft inside your cur
 Installing Playwright Pytest from Virtual Environment
 
 - To install libraries run "pip install -r requirements.txt"
-- Install the required browsers "playwright install"
+
 
 Run test cases
 
-- To run test cases from your 'tests' folder : "pytest --html=report.html --self-contained-html"
+- To run test cases from your 'tests/e2e-test' folder : "pytest --html=report.html --self-contained-html"
 
 Create .env file in project root level with web app url and client credentials
 

tests/e2e-test/base/base.py

@@ -45,23 +45,4 @@ def validate_response_status(self, question_api=""):
             "response code is " + str(response.status) + " " + str(response.json())
         )
 
-    def validate_response_status_draft_section(self, sectionTitle):
-        load_dotenv()  # Ensure environment variables are loaded
-        # URL of the API endpoint
-        url = f"{URL}/section/generate"
-
-        # Prepare headers
-        headers = {
-            "Content-Type": "application/json",
-            "Accept": "*/*",
-        }
-
-        payload = {"sectionTitle": sectionTitle}
-
-        # Make the POST request
-        response = self.page.request.post(
-            url, headers=headers, data=json.dumps(payload), timeout=200000
-        )
-        assert response.status == 200, (
-            "response code is " + str(response.status) + " " + str(response.json())
-        )
+  

tests/e2e-test/pages/draftPage.py

@@ -1,5 +1,7 @@
 from base.base import BasePage
 from pytest_check import check
+import time
+from collections import defaultdict
 
 
 class DraftPage(BasePage):
@@ -15,25 +17,74 @@ class DraftPage(BasePage):
     def __init__(self, page):
         self.page = page
 
-    def check_draft_Sections(self):
-        self.page.wait_for_timeout(10000)
-        if self.page.locator(self.Draft_Sections).count() >= 1:
-            for i in range(self.page.locator(self.Draft_Sections).count()):
-                draft_sections_response = self.page.locator(self.Draft_Sections).nth(i)
-                draft_heading = (
-                    self.page.locator(self.Draft_headings).nth(i).text_content()
-                )
-                check.not_equal(
-                    self.invalid_response,
-                    draft_sections_response.text_content(),
-                    f"Invalid response for {draft_heading} section",
-                )
-                check.not_equal(
-                    self.invalid_response1,
-                    draft_sections_response.text_content(),
-                    f"Invalid response for {draft_heading} section",
-                )
-                check.is_not_none(
-                    draft_sections_response.text_content(),
-                    f"Invalid response for {draft_heading} section",
-                )
+    def check_draft_sections(self, timeout: float = 180.0, poll_interval: float = 0.5):
+        """
+        Waits for all <textarea> draft sections to load valid content using .input_value().
+        Scrolls into view if needed, retries until timeout.
+        Raises clear errors if validation fails.
+        """
+        from collections import defaultdict
+        import time
+        start_time = time.time()
+
+
+        while time.time() - start_time < timeout:
+            section_elements = self.page.locator(self.Draft_Sections)
+            heading_elements = self.page.locator(self.Draft_headings)
+
+            section_count = section_elements.count()
+            heading_count = heading_elements.count()
+
+            if section_count < 13 or heading_count < 13:
+                print("[WAIT] Waiting for all sections to appear...")
+                time.sleep(poll_interval)
+                continue
+
+            failed_sections = defaultdict(str)
+
+            for i in range(section_count):
+                section = section_elements.nth(i)
+
+                try:
+                    # Scroll into view and wait a bit for rendering
+                    section.scroll_into_view_if_needed(timeout=2000)
+                    self.page.wait_for_timeout(200)
+
+                    # Extract content from <textarea>
+                    section_text = section.input_value().strip()
+
+                    if not section_text:
+                        failed_sections[i] = "Empty"
+                    elif section_text in (self.invalid_response, self.invalid_response1):
+                        failed_sections[i] = f"Invalid: {repr(section_text[:30])}"
+
+                except Exception as e:
+                    failed_sections[i] = f"Exception: {str(e)}"
+
+            if not failed_sections:
+                break  # ✅ All good
+            else:
+                print(f"[WAITING] Sections not ready yet: {failed_sections}")
+                time.sleep(poll_interval)
+
+        else:
+            raise TimeoutError(f"❌ Timeout: These sections did not load valid content: {failed_sections}")
+
+        # ✅ Final validations after loading
+        for i in range(section_count):
+            section = section_elements.nth(i)
+            heading = heading_elements.nth(i)
+
+            section.scroll_into_view_if_needed(timeout=2000)
+            self.page.wait_for_timeout(200)
+
+            heading_text = heading.inner_text(timeout=3000).strip()
+            content = section.input_value().strip()
+
+            print(f"[VALIDATING] Section {i}: '{heading_text}' → {repr(content[:60])}...")
+
+            with check:
+                check.is_not_none(content, f"❌ Section '{heading_text}' is None")
+                check.not_equal(content, "", f"❌ Section '{heading_text}' is empty")
+                check.not_equal(content, self.invalid_response, f"❌ '{heading_text}' has invalid response")
+                check.not_equal(content, self.invalid_response1, f"❌ '{heading_text}' has invalid response")

tests/e2e-test/pages/generatePage.py

@@ -1,18 +1,29 @@
 from base.base import BasePage
-
+from playwright.sync_api import expect
+from asyncio.log import logger
 
 class GeneratePage(BasePage):
     GENERATE_DRAFT = "//button[@title='Generate Draft']"
     TYPE_QUESTION = "//textarea[@placeholder='Type a new question...']"
     SEND_BUTTON = "//div[@aria-label='Ask question button']"
+    SHOW_CHAT_HISTORY_BUTTON="//span[text()='Show template history']"
+    HIDE_CHAT_HISTORY_BUTTON = "//span[text()='Hide Chat History']"
+    CHAT_HISTORY_ITEM = "//body//div[@id='root']//div[@role='presentation']//div[@role='presentation']//div[1]//div[1]//div[1]//div[1]//div[1]//div[1]"
+    SHOW_CHAT_HISTORY = "//span//i"
+    CHAT_HISTORY_NAME = "div[aria-label='chat history list']"
+    CHAT_CLOSE_ICON = "button[title='Hide']"
+    CHAT_HISTORY_OPTIONS = "//button[@id='moreButton']"
+    CHAT_HISTORY_DELETE = "//button[@role='menuitem']"
+    CHAT_HISTORY_CLOSE = "//i[@data-icon-name='Cancel']"
 
     def __init__(self, page):
         self.page = page
 
     def enter_a_question(self, text):
         # Type a question in the text area
+        self.page.wait_for_timeout(3000)
         self.page.locator(self.TYPE_QUESTION).fill(text)
-        self.page.wait_for_timeout(2000)
+        self.page.wait_for_timeout(3000)
 
     def click_send_button(self):
         # Type a question in the text area
@@ -23,3 +34,41 @@ def click_generate_draft_button(self):
         # Type a question in the text area
         self.page.locator(self.GENERATE_DRAFT).click()
         self.page.wait_for_timeout(15000)
+
+    def show_chat_history(self):
+        """Click to show chat history if the button is visible."""
+        show_button = self.page.locator(self.SHOW_CHAT_HISTORY_BUTTON)
+        if show_button.is_visible():
+            show_button.click()
+            self.page.wait_for_timeout(2000)
+            expect(self.page.locator(self.CHAT_HISTORY_ITEM)).to_be_visible()
+        else:
+            logger.info("Chat history is not generated")
+
+    def close_chat_history(self):
+        """Click to close chat history if visible."""
+        hide_button = self.page.locator(self.HIDE_CHAT_HISTORY_BUTTON)
+        if hide_button.is_visible():
+            hide_button.click()
+            self.page.wait_for_timeout(2000)
+        else:
+            logger.info("Hide button not visible. Chat history might already be closed.")
+
+    def delete_chat_history(self):
+
+        self.page.locator(self.SHOW_CHAT_HISTORY_BUTTON).click()
+        chat_history = self.page.locator("//span[contains(text(),'No chat history.')]")
+        if chat_history.is_visible():
+            self.page.wait_for_load_state("networkidle")
+            self.page.locator("button[title='Hide']").wait_for(state="visible", timeout=5000)
+            self.page.locator("button[title='Hide']").click()
+
+        else:
+            self.page.locator(self.CHAT_HISTORY_OPTIONS).click()
+            self.page.locator(self.CHAT_HISTORY_DELETE).click()
+            self.page.get_by_role("button", name="Clear All").click()
+            self.page.wait_for_timeout(5000)
+            self.page.locator(self.CHAT_HISTORY_CLOSE).click()
+            self.page.wait_for_load_state("networkidle")
+            self.page.wait_for_timeout(2000)
+

tests/e2e-test/pages/homePage.py

@@ -24,6 +24,7 @@ def click_browse_button(self):
         self.page.wait_for_timeout(5000)
 
     def validate_home_page(self):
+        self.page.wait_for_timeout(5000)
         expect(self.page.locator(self.HOME_TITLE)).to_be_visible()
         expect(self.page.locator(self.BROWSE_TEXT)).to_be_visible()
         expect(self.page.locator(self.GENERATE_TEXT)).to_be_visible()

tests/e2e-test/requirements.txt

@@ -3,4 +3,5 @@ pytest-reporter-html1
 python-dotenv
 pytest-check
 pytest-html
-py
+py
+beautifulsoup4

tests/e2e-test/testData/section_title.json

@@ -1,9 +1,15 @@
-import os
-
+from pathlib import Path
 import pytest
-from config.constants import URL
 from playwright.sync_api import sync_playwright
-from py.xml import html  # type: ignore
+from config.constants import *
+from slugify import slugify
+from dotenv import load_dotenv
+import os
+from py.xml import html # type: ignore
+import io
+import logging
+from bs4 import BeautifulSoup
+import atexit
 
 
 @pytest.fixture(scope="session")
@@ -13,11 +19,16 @@ def login_logout():
         browser = p.chromium.launch(headless=False, args=["--start-maximized"])
         context = browser.new_context(no_viewport=True)
         context.set_default_timeout(120000)
+        context.clear_cookies()
         page = context.new_page()
         # Navigate to the login URL
         page.goto(URL)
         # Wait for the login form to appear
         # page.wait_for_load_state('networkidle')
+        # login to web url with username and password
+        #login_page = LoginPage(page)
+        #load_dotenv()
+        #login_page.authenticate(os.getenv('user_name'),os.getenv('pass_word'))
         yield page
 
         # perform close the browser
@@ -26,26 +37,77 @@ def login_logout():
 
 @pytest.hookimpl(tryfirst=True)
 def pytest_html_report_title(report):
-    report.title = "Automation_DOCGEN"
+    report.title = "Test Automation DocGen"
+
 
+log_streams = {}
 
-# Add a column for descriptions
-def pytest_html_results_table_header(cells):
-    cells.insert(1, html.th("Description"))
+@pytest.hookimpl(tryfirst=True)
+def pytest_runtest_setup(item):
+    # Prepare StringIO for capturing logs
+    stream = io.StringIO()
+    handler = logging.StreamHandler(stream)
+    handler.setLevel(logging.INFO)
 
+    logger = logging.getLogger()
+    logger.addHandler(handler)
 
-def pytest_html_results_table_row(report, cells):
-    cells.insert(
-        1, html.td(report.description if hasattr(report, "description") else "")
-    )
+    # Save handler and stream
+    log_streams[item.nodeid] = (handler, stream)
 
 
-# Add logs and docstring to report
 @pytest.hookimpl(hookwrapper=True)
 def pytest_runtest_makereport(item, call):
     outcome = yield
     report = outcome.get_result()
-    report.description = str(item.function.__doc__)
-    os.makedirs("logs", exist_ok=True)
-    extra = getattr(report, "extra", [])
-    report.extra = extra
+
+    handler, stream = log_streams.get(item.nodeid, (None, None))
+
+    if handler and stream:
+        # Make sure logs are flushed
+        handler.flush()
+        log_output = stream.getvalue()
+
+        # Only remove the handler, don't close the stream yet
+        logger = logging.getLogger()
+        logger.removeHandler(handler)
+
+        # Store the log output on the report object for HTML reporting
+        report.description = f"<pre>{log_output.strip()}</pre>"
+
+        # Clean up references
+        log_streams.pop(item.nodeid, None)
+    else:
+        report.description = ""
+
+def pytest_collection_modifyitems(items):
+    for item in items:
+        if hasattr(item, 'callspec'):
+            prompt = item.callspec.params.get("prompt")
+            if prompt:
+                item._nodeid = prompt  # This controls how the test name appears in the report
+
+def rename_duration_column():
+    report_path = os.path.abspath("report.html")  # or your report filename
+    if not os.path.exists(report_path):
+        print("Report file not found, skipping column rename.")
+        return
+
+    with open(report_path, 'r', encoding='utf-8') as f:
+        soup = BeautifulSoup(f, 'html.parser')
+
+    # Find and rename the header
+    headers = soup.select('table#results-table thead th')
+    for th in headers:
+        if th.text.strip() == 'Duration':
+            th.string = 'Execution Time'
+            #print("Renamed 'Duration' to 'Execution Time'")
+            break
+    else:
+        print("'Duration' column not found in report.")
+
+    with open(report_path, 'w', encoding='utf-8') as f:
+        f.write(str(soup))
+
+# Register this function to run after everything is done
+atexit.register(rename_duration_column)