test: MACAE-Implemented Log execution time per prompt in Report

Author: Avijit-Microsoft

tests/e2e-test/README.md

@@ -20,11 +20,11 @@ This will create a virtual environment directory named microsoft inside your cur
 Installing Playwright Pytest from Virtual Environment
 
 - To install libraries run "pip install -r requirements.txt"
-- Install the required browsers "playwright install"
+
 
 Run test cases
 
-- To run test cases from your 'tests' folder : "pytest --headed --html=report/report.html"
+- To run test cases from your 'tests/e2e-test' folder : "pytest --html=report.html --self-contained-html"
 
 Create .env file in project root level with web app url and client credentials
 

tests/e2e-test/requirements.txt

@@ -4,3 +4,4 @@ python-dotenv
 pytest-check
 pytest-html
 py
+beautifulsoup4

tests/e2e-test/tests/conftest.py

@@ -1,9 +1,16 @@
-import os
-
+from pathlib import Path
 import pytest
-from config.constants import URL
 from playwright.sync_api import sync_playwright
-from py.xml import html  # type: ignore
+from config.constants import *
+from slugify import slugify
+from pages.loginPage import LoginPage
+from dotenv import load_dotenv
+import os
+from py.xml import html # type: ignore
+import io
+import logging
+from bs4 import BeautifulSoup
+import atexit
 
 
 @pytest.fixture(scope="session")
@@ -17,8 +24,11 @@ def login_logout():
         # Navigate to the login URL
         page.goto(URL)
         # Wait for the login form to appear
-        page.wait_for_load_state("networkidle")
-
+        page.wait_for_load_state('networkidle')
+        # login to web url with username and password
+        #login_page = LoginPage(page)
+        #load_dotenv()
+        #login_page.authenticate(os.getenv('user_name'),os.getenv('pass_word'))
         yield page
 
         # perform close the browser
@@ -27,26 +37,77 @@ def login_logout():
 
 @pytest.hookimpl(tryfirst=True)
 def pytest_html_report_title(report):
-    report.title = "Automation_MACAE"
+    report.title = "Test Automation MACAE"
 
 
-# Add a column for descriptions
-def pytest_html_results_table_header(cells):
-    cells.insert(1, html.th("Description"))
+log_streams = {}
 
+@pytest.hookimpl(tryfirst=True)
+def pytest_runtest_setup(item):
+    # Prepare StringIO for capturing logs
+    stream = io.StringIO()
+    handler = logging.StreamHandler(stream)
+    handler.setLevel(logging.INFO)
 
-def pytest_html_results_table_row(report, cells):
-    cells.insert(
-        1, html.td(report.description if hasattr(report, "description") else "")
-    )
+    logger = logging.getLogger()
+    logger.addHandler(handler)
+
+    # Save handler and stream
+    log_streams[item.nodeid] = (handler, stream)
 
 
-# Add logs and docstring to report
 @pytest.hookimpl(hookwrapper=True)
 def pytest_runtest_makereport(item, call):
     outcome = yield
     report = outcome.get_result()
-    report.description = str(item.function.__doc__)
-    os.makedirs("logs", exist_ok=True)
-    extra = getattr(report, "extra", [])
-    report.extra = extra
+
+    handler, stream = log_streams.get(item.nodeid, (None, None))
+
+    if handler and stream:
+        # Make sure logs are flushed
+        handler.flush()
+        log_output = stream.getvalue()
+
+        # Only remove the handler, don't close the stream yet
+        logger = logging.getLogger()
+        logger.removeHandler(handler)
+
+        # Store the log output on the report object for HTML reporting
+        report.description = f"<pre>{log_output.strip()}</pre>"
+
+        # Clean up references
+        log_streams.pop(item.nodeid, None)
+    else:
+        report.description = ""
+
+def pytest_collection_modifyitems(items):
+    for item in items:
+        if hasattr(item, 'callspec'):
+            prompt = item.callspec.params.get("prompt")
+            if prompt:
+                item._nodeid = prompt  # This controls how the test name appears in the report
+
+def rename_duration_column():
+    report_path = os.path.abspath("report.html")  # or your report filename
+    if not os.path.exists(report_path):
+        print("Report file not found, skipping column rename.")
+        return
+
+    with open(report_path, 'r', encoding='utf-8') as f:
+        soup = BeautifulSoup(f, 'html.parser')
+
+    # Find and rename the header
+    headers = soup.select('table#results-table thead th')
+    for th in headers:
+        if th.text.strip() == 'Duration':
+            th.string = 'Execution Time'
+            #print("Renamed 'Duration' to 'Execution Time'")
+            break
+    else:
+        print("'Duration' column not found in report.")
+
+    with open(report_path, 'w', encoding='utf-8') as f:
+        f.write(str(soup))
+
+# Register this function to run after everything is done
+atexit.register(rename_duration_column)

tests/e2e-test/tests/test_MACAE_GP.py

@@ -0,0 +1,83 @@
+import logging
+import time
+import pytest
+from pages.BIAB import BIABPage
+from config.constants import (
+    prompt_question1,
+    prompt_question2,
+    rai_prompt,
+    employee_details,
+    product_details,
+)
+
+logger = logging.getLogger(__name__)
+
+
+# Define test steps and prompts
+test_cases = [
+    ("Validate home page is loaded", lambda biab: biab.validate_home_page()),
+
+    (f"Verify Run Prompt 1: '{prompt_question1}' & run all stages", lambda biab: (
+        biab.enter_a_question(prompt_question1),
+        biab.click_send_button(),
+        biab.click_my_task(),
+        biab.enter_aditional_info(employee_details),
+        # biab.click_aditional_send_button(),
+        biab.processing_different_stage(),
+        biab.click_new_task()
+    )),
+
+    (f"Verify Run Prompt 2: '{prompt_question2}' & run all stages", lambda biab: (
+        biab.enter_a_question(prompt_question2),
+        biab.click_send_button(),
+        biab.click_my_task(),
+        biab.enter_aditional_info(product_details),
+        # biab.click_aditional_send_button(),
+        biab.processing_different_stage(),
+        biab.click_new_task()
+    )),
+
+    ("Verify Run Prompt 3 via Quick Task - Mobile Plan Query & run all stages", lambda biab: (
+        biab.click_mobile_plan(),
+        biab.click_send_button(),
+        biab.click_my_task(),
+        biab.processing_different_stage(),
+        biab.click_new_task()
+    )),
+
+    (f"Verify Run RAI Prompt: '{rai_prompt}' to make sure task is not created and validation message is displayed.", lambda biab: (
+        biab.enter_a_question(rai_prompt),
+        biab.validate_rai_validation_message()
+    )),
+]
+
+# Create test IDs like "01. Validate home page", "02. Run Prompt 1: ..."
+test_ids = [f"{i+1:02d}. {case[0]}" for i, case in enumerate(test_cases)]
+
+
+@pytest.mark.parametrize("prompt, action", test_cases, ids=test_ids)
+def test_biab_prompt_case(login_logout, prompt, action, request):
+    """
+    Each BIAB prompt runs as an individual test case with execution time logging
+    and meaningful test step titles.
+    """
+    page = login_logout
+    biab_page = BIABPage(page)
+    logger.info(f"Running test step: {prompt}")
+
+    start = time.time()
+    if isinstance(action, tuple):
+        for step in action:
+            if callable(step):
+                step()
+    else:
+        action(biab_page)
+    end = time.time()
+
+    duration = end - start
+    logger.info(f"Execution Time for '{prompt}': {duration:.2f}s")
+
+    # Attach execution time to pytest report
+    request.node._report_sections.append((
+        "call", "log", f"Execution time: {duration:.2f}s"
+    ))