diff --git a/tests/e2e-test/README.md b/tests/e2e-test/README.md index 5518e3f01..43277195a 100644 --- a/tests/e2e-test/README.md +++ b/tests/e2e-test/README.md @@ -20,11 +20,11 @@ This will create a virtual environment directory named microsoft inside your cur Installing Playwright Pytest from Virtual Environment - To install libraries run "pip install -r requirements.txt" -- Install the required browsers "playwright install" + Run test cases -- To run test cases from your 'tests' folder : "pytest --headed --html=report/report.html" +- To run test cases from your 'tests/e2e-test' folder : "pytest --html=report.html --self-contained-html" Create .env file in project root level with web app url and client credentials diff --git a/tests/e2e-test/requirements.txt b/tests/e2e-test/requirements.txt index 1b0ac0d7c..4e488e559 100644 --- a/tests/e2e-test/requirements.txt +++ b/tests/e2e-test/requirements.txt @@ -4,3 +4,4 @@ python-dotenv pytest-check pytest-html py +beautifulsoup4 \ No newline at end of file diff --git a/tests/e2e-test/tests/conftest.py b/tests/e2e-test/tests/conftest.py index 92e34ccaa..f09dd92ed 100644 --- a/tests/e2e-test/tests/conftest.py +++ b/tests/e2e-test/tests/conftest.py @@ -1,9 +1,16 @@ -import os - +from pathlib import Path import pytest -from config.constants import URL from playwright.sync_api import sync_playwright -from py.xml import html # type: ignore +from config.constants import * +from slugify import slugify +from pages.loginPage import LoginPage +from dotenv import load_dotenv +import os +from py.xml import html # type: ignore +import io +import logging +from bs4 import BeautifulSoup +import atexit @pytest.fixture(scope="session") @@ -17,8 +24,11 @@ def login_logout(): # Navigate to the login URL page.goto(URL) # Wait for the login form to appear - page.wait_for_load_state("networkidle") - + page.wait_for_load_state('networkidle') + # login to web url with username and password + #login_page = LoginPage(page) + #load_dotenv() + #login_page.authenticate(os.getenv('user_name'),os.getenv('pass_word')) yield page # perform close the browser @@ -27,26 +37,77 @@ def login_logout(): @pytest.hookimpl(tryfirst=True) def pytest_html_report_title(report): - report.title = "Automation_MACAE" + report.title = "Test Automation MACAE" -# Add a column for descriptions -def pytest_html_results_table_header(cells): - cells.insert(1, html.th("Description")) +log_streams = {} +@pytest.hookimpl(tryfirst=True) +def pytest_runtest_setup(item): + # Prepare StringIO for capturing logs + stream = io.StringIO() + handler = logging.StreamHandler(stream) + handler.setLevel(logging.INFO) -def pytest_html_results_table_row(report, cells): - cells.insert( - 1, html.td(report.description if hasattr(report, "description") else "") - ) + logger = logging.getLogger() + logger.addHandler(handler) + + # Save handler and stream + log_streams[item.nodeid] = (handler, stream) -# Add logs and docstring to report @pytest.hookimpl(hookwrapper=True) def pytest_runtest_makereport(item, call): outcome = yield report = outcome.get_result() - report.description = str(item.function.__doc__) - os.makedirs("logs", exist_ok=True) - extra = getattr(report, "extra", []) - report.extra = extra + + handler, stream = log_streams.get(item.nodeid, (None, None)) + + if handler and stream: + # Make sure logs are flushed + handler.flush() + log_output = stream.getvalue() + + # Only remove the handler, don't close the stream yet + logger = logging.getLogger() + logger.removeHandler(handler) + + # Store the log output on the report object for HTML reporting + report.description = f"
{log_output.strip()}"
+
+ # Clean up references
+ log_streams.pop(item.nodeid, None)
+ else:
+ report.description = ""
+
+def pytest_collection_modifyitems(items):
+ for item in items:
+ if hasattr(item, 'callspec'):
+ prompt = item.callspec.params.get("prompt")
+ if prompt:
+ item._nodeid = prompt # This controls how the test name appears in the report
+
+def rename_duration_column():
+ report_path = os.path.abspath("report.html") # or your report filename
+ if not os.path.exists(report_path):
+ print("Report file not found, skipping column rename.")
+ return
+
+ with open(report_path, 'r', encoding='utf-8') as f:
+ soup = BeautifulSoup(f, 'html.parser')
+
+ # Find and rename the header
+ headers = soup.select('table#results-table thead th')
+ for th in headers:
+ if th.text.strip() == 'Duration':
+ th.string = 'Execution Time'
+ #print("Renamed 'Duration' to 'Execution Time'")
+ break
+ else:
+ print("'Duration' column not found in report.")
+
+ with open(report_path, 'w', encoding='utf-8') as f:
+ f.write(str(soup))
+
+# Register this function to run after everything is done
+atexit.register(rename_duration_column)
\ No newline at end of file
diff --git a/tests/e2e-test/tests/test_MACAE_GP.py b/tests/e2e-test/tests/test_MACAE_GP.py
new file mode 100644
index 000000000..e9c71f55c
--- /dev/null
+++ b/tests/e2e-test/tests/test_MACAE_GP.py
@@ -0,0 +1,83 @@
+import logging
+import time
+import pytest
+from pages.BIAB import BIABPage
+from config.constants import (
+ prompt_question1,
+ prompt_question2,
+ rai_prompt,
+ employee_details,
+ product_details,
+)
+
+logger = logging.getLogger(__name__)
+
+
+# Define test steps and prompts
+test_cases = [
+ ("Validate home page is loaded", lambda biab: biab.validate_home_page()),
+
+ (f"Verify Run Prompt 1: '{prompt_question1}' & run all stages", lambda biab: (
+ biab.enter_a_question(prompt_question1),
+ biab.click_send_button(),
+ biab.click_my_task(),
+ biab.enter_aditional_info(employee_details),
+ # biab.click_aditional_send_button(),
+ biab.processing_different_stage(),
+ biab.click_new_task()
+ )),
+
+ (f"Verify Run Prompt 2: '{prompt_question2}' & run all stages", lambda biab: (
+ biab.enter_a_question(prompt_question2),
+ biab.click_send_button(),
+ biab.click_my_task(),
+ biab.enter_aditional_info(product_details),
+ # biab.click_aditional_send_button(),
+ biab.processing_different_stage(),
+ biab.click_new_task()
+ )),
+
+ ("Verify Run Prompt 3 via Quick Task - Mobile Plan Query & run all stages", lambda biab: (
+ biab.click_mobile_plan(),
+ biab.click_send_button(),
+ biab.click_my_task(),
+ biab.processing_different_stage(),
+ biab.click_new_task()
+ )),
+
+ (f"Verify Run RAI Prompt: '{rai_prompt}' to make sure task is not created and validation message is displayed.", lambda biab: (
+ biab.enter_a_question(rai_prompt),
+ biab.validate_rai_validation_message()
+ )),
+]
+
+# Create test IDs like "01. Validate home page", "02. Run Prompt 1: ..."
+test_ids = [f"{i+1:02d}. {case[0]}" for i, case in enumerate(test_cases)]
+
+
+@pytest.mark.parametrize("prompt, action", test_cases, ids=test_ids)
+def test_biab_prompt_case(login_logout, prompt, action, request):
+ """
+ Each BIAB prompt runs as an individual test case with execution time logging
+ and meaningful test step titles.
+ """
+ page = login_logout
+ biab_page = BIABPage(page)
+ logger.info(f"Running test step: {prompt}")
+
+ start = time.time()
+ if isinstance(action, tuple):
+ for step in action:
+ if callable(step):
+ step()
+ else:
+ action(biab_page)
+ end = time.time()
+
+ duration = end - start
+ logger.info(f"Execution Time for '{prompt}': {duration:.2f}s")
+
+ # Attach execution time to pytest report
+ request.node._report_sections.append((
+ "call", "log", f"Execution time: {duration:.2f}s"
+ ))
diff --git a/tests/e2e-test/tests/test_poc_BIAB.py b/tests/e2e-test/tests/test_poc_BIAB.py
deleted file mode 100644
index b382146ad..000000000
--- a/tests/e2e-test/tests/test_poc_BIAB.py
+++ /dev/null
@@ -1,41 +0,0 @@
-import logging
-
-from config.constants import prompt_question1, prompt_question2, rai_prompt, employee_details, product_details
-from pages.BIAB import BIABPage
-
-logger = logging.getLogger(__name__)
-
-
-def test_biab_PAGE_AUTOMATION(login_logout):
- """Validate Golden path test case for Multi-Agent-Custom-Automation-Engine"""
- page = login_logout
- biab_page = BIABPage(page)
- logger.info("Step 1: Validate home page is loaded.")
- biab_page.validate_home_page()
- logger.info("Step 2: Validate Run Sample prompt1 & run plans")
- biab_page.enter_a_question(prompt_question1)
- biab_page.click_send_button()
- biab_page.click_my_task()
- biab_page.enter_aditional_info(employee_details)
- # biab_page.click_aditional_send_button()
- biab_page.processing_different_stage()
- biab_page.click_new_task()
- logger.info("Step 3: Validate Run Sample prompt2 & run plans")
- biab_page.enter_a_question(prompt_question2)
- biab_page.click_send_button()
- biab_page.click_my_task()
- biab_page.enter_aditional_info(product_details)
- # biab_page.click_aditional_send_button()
- biab_page.processing_different_stage()
- biab_page.click_new_task()
- logger.info("Step 4: Validate Run Sample prompt3 from Quick Tasks & run plans")
- biab_page.click_mobile_plan()
- biab_page.click_send_button()
- biab_page.click_my_task()
- biab_page.processing_different_stage()
- biab_page.click_new_task()
- logger.info(
- "Step 5: Validate Run known RAI test prompts to ensure that you get the toast saying that a plan cannot be generated"
- )
- biab_page.enter_a_question(rai_prompt)
- biab_page.validate_rai_validation_message()