diff --git a/tests/e2e-test/README.md b/tests/e2e-test/README.md index 5518e3f01..43277195a 100644 --- a/tests/e2e-test/README.md +++ b/tests/e2e-test/README.md @@ -20,11 +20,11 @@ This will create a virtual environment directory named microsoft inside your cur Installing Playwright Pytest from Virtual Environment - To install libraries run "pip install -r requirements.txt" -- Install the required browsers "playwright install" + Run test cases -- To run test cases from your 'tests' folder : "pytest --headed --html=report/report.html" +- To run test cases from your 'tests/e2e-test' folder : "pytest --html=report.html --self-contained-html" Create .env file in project root level with web app url and client credentials diff --git a/tests/e2e-test/requirements.txt b/tests/e2e-test/requirements.txt index 1b0ac0d7c..4e488e559 100644 --- a/tests/e2e-test/requirements.txt +++ b/tests/e2e-test/requirements.txt @@ -4,3 +4,4 @@ python-dotenv pytest-check pytest-html py +beautifulsoup4 \ No newline at end of file diff --git a/tests/e2e-test/tests/conftest.py b/tests/e2e-test/tests/conftest.py index 92e34ccaa..f09dd92ed 100644 --- a/tests/e2e-test/tests/conftest.py +++ b/tests/e2e-test/tests/conftest.py @@ -1,9 +1,16 @@ -import os - +from pathlib import Path import pytest -from config.constants import URL from playwright.sync_api import sync_playwright -from py.xml import html # type: ignore +from config.constants import * +from slugify import slugify +from pages.loginPage import LoginPage +from dotenv import load_dotenv +import os +from py.xml import html # type: ignore +import io +import logging +from bs4 import BeautifulSoup +import atexit @pytest.fixture(scope="session") @@ -17,8 +24,11 @@ def login_logout(): # Navigate to the login URL page.goto(URL) # Wait for the login form to appear - page.wait_for_load_state("networkidle") - + page.wait_for_load_state('networkidle') + # login to web url with username and password + #login_page = LoginPage(page) + #load_dotenv() + #login_page.authenticate(os.getenv('user_name'),os.getenv('pass_word')) yield page # perform close the browser @@ -27,26 +37,77 @@ def login_logout(): @pytest.hookimpl(tryfirst=True) def pytest_html_report_title(report): - report.title = "Automation_MACAE" + report.title = "Test Automation MACAE" -# Add a column for descriptions -def pytest_html_results_table_header(cells): - cells.insert(1, html.th("Description")) +log_streams = {} +@pytest.hookimpl(tryfirst=True) +def pytest_runtest_setup(item): + # Prepare StringIO for capturing logs + stream = io.StringIO() + handler = logging.StreamHandler(stream) + handler.setLevel(logging.INFO) -def pytest_html_results_table_row(report, cells): - cells.insert( - 1, html.td(report.description if hasattr(report, "description") else "") - ) + logger = logging.getLogger() + logger.addHandler(handler) + + # Save handler and stream + log_streams[item.nodeid] = (handler, stream) -# Add logs and docstring to report @pytest.hookimpl(hookwrapper=True) def pytest_runtest_makereport(item, call): outcome = yield report = outcome.get_result() - report.description = str(item.function.__doc__) - os.makedirs("logs", exist_ok=True) - extra = getattr(report, "extra", []) - report.extra = extra + + handler, stream = log_streams.get(item.nodeid, (None, None)) + + if handler and stream: + # Make sure logs are flushed + handler.flush() + log_output = stream.getvalue() + + # Only remove the handler, don't close the stream yet + logger = logging.getLogger() + logger.removeHandler(handler) + + # Store the log output on the report object for HTML reporting + report.description = f"
{log_output.strip()}
" + + # Clean up references + log_streams.pop(item.nodeid, None) + else: + report.description = "" + +def pytest_collection_modifyitems(items): + for item in items: + if hasattr(item, 'callspec'): + prompt = item.callspec.params.get("prompt") + if prompt: + item._nodeid = prompt # This controls how the test name appears in the report + +def rename_duration_column(): + report_path = os.path.abspath("report.html") # or your report filename + if not os.path.exists(report_path): + print("Report file not found, skipping column rename.") + return + + with open(report_path, 'r', encoding='utf-8') as f: + soup = BeautifulSoup(f, 'html.parser') + + # Find and rename the header + headers = soup.select('table#results-table thead th') + for th in headers: + if th.text.strip() == 'Duration': + th.string = 'Execution Time' + #print("Renamed 'Duration' to 'Execution Time'") + break + else: + print("'Duration' column not found in report.") + + with open(report_path, 'w', encoding='utf-8') as f: + f.write(str(soup)) + +# Register this function to run after everything is done +atexit.register(rename_duration_column) \ No newline at end of file diff --git a/tests/e2e-test/tests/test_MACAE_GP.py b/tests/e2e-test/tests/test_MACAE_GP.py new file mode 100644 index 000000000..e9c71f55c --- /dev/null +++ b/tests/e2e-test/tests/test_MACAE_GP.py @@ -0,0 +1,83 @@ +import logging +import time +import pytest +from pages.BIAB import BIABPage +from config.constants import ( + prompt_question1, + prompt_question2, + rai_prompt, + employee_details, + product_details, +) + +logger = logging.getLogger(__name__) + + +# Define test steps and prompts +test_cases = [ + ("Validate home page is loaded", lambda biab: biab.validate_home_page()), + + (f"Verify Run Prompt 1: '{prompt_question1}' & run all stages", lambda biab: ( + biab.enter_a_question(prompt_question1), + biab.click_send_button(), + biab.click_my_task(), + biab.enter_aditional_info(employee_details), + # biab.click_aditional_send_button(), + biab.processing_different_stage(), + biab.click_new_task() + )), + + (f"Verify Run Prompt 2: '{prompt_question2}' & run all stages", lambda biab: ( + biab.enter_a_question(prompt_question2), + biab.click_send_button(), + biab.click_my_task(), + biab.enter_aditional_info(product_details), + # biab.click_aditional_send_button(), + biab.processing_different_stage(), + biab.click_new_task() + )), + + ("Verify Run Prompt 3 via Quick Task - Mobile Plan Query & run all stages", lambda biab: ( + biab.click_mobile_plan(), + biab.click_send_button(), + biab.click_my_task(), + biab.processing_different_stage(), + biab.click_new_task() + )), + + (f"Verify Run RAI Prompt: '{rai_prompt}' to make sure task is not created and validation message is displayed.", lambda biab: ( + biab.enter_a_question(rai_prompt), + biab.validate_rai_validation_message() + )), +] + +# Create test IDs like "01. Validate home page", "02. Run Prompt 1: ..." +test_ids = [f"{i+1:02d}. {case[0]}" for i, case in enumerate(test_cases)] + + +@pytest.mark.parametrize("prompt, action", test_cases, ids=test_ids) +def test_biab_prompt_case(login_logout, prompt, action, request): + """ + Each BIAB prompt runs as an individual test case with execution time logging + and meaningful test step titles. + """ + page = login_logout + biab_page = BIABPage(page) + logger.info(f"Running test step: {prompt}") + + start = time.time() + if isinstance(action, tuple): + for step in action: + if callable(step): + step() + else: + action(biab_page) + end = time.time() + + duration = end - start + logger.info(f"Execution Time for '{prompt}': {duration:.2f}s") + + # Attach execution time to pytest report + request.node._report_sections.append(( + "call", "log", f"Execution time: {duration:.2f}s" + )) diff --git a/tests/e2e-test/tests/test_poc_BIAB.py b/tests/e2e-test/tests/test_poc_BIAB.py deleted file mode 100644 index b382146ad..000000000 --- a/tests/e2e-test/tests/test_poc_BIAB.py +++ /dev/null @@ -1,41 +0,0 @@ -import logging - -from config.constants import prompt_question1, prompt_question2, rai_prompt, employee_details, product_details -from pages.BIAB import BIABPage - -logger = logging.getLogger(__name__) - - -def test_biab_PAGE_AUTOMATION(login_logout): - """Validate Golden path test case for Multi-Agent-Custom-Automation-Engine""" - page = login_logout - biab_page = BIABPage(page) - logger.info("Step 1: Validate home page is loaded.") - biab_page.validate_home_page() - logger.info("Step 2: Validate Run Sample prompt1 & run plans") - biab_page.enter_a_question(prompt_question1) - biab_page.click_send_button() - biab_page.click_my_task() - biab_page.enter_aditional_info(employee_details) - # biab_page.click_aditional_send_button() - biab_page.processing_different_stage() - biab_page.click_new_task() - logger.info("Step 3: Validate Run Sample prompt2 & run plans") - biab_page.enter_a_question(prompt_question2) - biab_page.click_send_button() - biab_page.click_my_task() - biab_page.enter_aditional_info(product_details) - # biab_page.click_aditional_send_button() - biab_page.processing_different_stage() - biab_page.click_new_task() - logger.info("Step 4: Validate Run Sample prompt3 from Quick Tasks & run plans") - biab_page.click_mobile_plan() - biab_page.click_send_button() - biab_page.click_my_task() - biab_page.processing_different_stage() - biab_page.click_new_task() - logger.info( - "Step 5: Validate Run known RAI test prompts to ensure that you get the toast saying that a plan cannot be generated" - ) - biab_page.enter_a_question(rai_prompt) - biab_page.validate_rai_validation_message()