|
| 1 | +# -*- coding: utf-8 -*- |
| 2 | +""" |
| 3 | +Copyright 2025 Telefónica Innovación Digital, S.L. |
| 4 | +This file is part of Toolium. |
| 5 | +
|
| 6 | +Licensed under the Apache License, Version 2.0 (the "License"); |
| 7 | +you may not use this file except in compliance with the License. |
| 8 | +You may obtain a copy of the License at |
| 9 | +
|
| 10 | + http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | +
|
| 12 | +Unless required by applicable law or agreed to in writing, software |
| 13 | +distributed under the License is distributed on an "AS IS" BASIS, |
| 14 | +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 15 | +See the License for the specific language governing permissions and |
| 16 | +limitations under the License. |
| 17 | +""" |
| 18 | + |
| 19 | +import functools |
| 20 | +import re |
| 21 | +from behave.model import ScenarioOutline |
| 22 | +from behave.model_core import Status |
| 23 | + |
| 24 | + |
| 25 | +def get_accuracy_and_retries_from_tags(tags): |
| 26 | + """ |
| 27 | + Extract accuracy and retries values from accuracy tag using regex. |
| 28 | + Examples of valid tags: |
| 29 | + - accuracy |
| 30 | + - accuracy_90 |
| 31 | + - accuracy_percent_90 |
| 32 | + - accuracy_90_10 |
| 33 | + - accuracy_percent_90_retries_10 |
| 34 | +
|
| 35 | + :param tags: behave tags |
| 36 | + :return: dict with 'accuracy' and 'retries' keys if tag matches, None otherwise |
| 37 | + """ |
| 38 | + accuracy_regex = re.compile(r'^accuracy(?:_(?:percent_)?(\d+)(?:_retries_(\d+)|_(\d+))?)?', re.IGNORECASE) |
| 39 | + for tag in tags: |
| 40 | + match = accuracy_regex.search(tag) |
| 41 | + if match: |
| 42 | + # Default values: 90% accuracy, 10 retries |
| 43 | + accuracy_percent = (int(match.group(1)) / 100.0) if match.group(1) else 0.9 |
| 44 | + # Check if retries is in group 2 (accuracy_percent_90_retries_10) or group 3 (accuracy_90_10) |
| 45 | + retries = int(match.group(2)) if match.group(2) else (int(match.group(3)) if match.group(3) else 10) |
| 46 | + return {'accuracy': accuracy_percent, 'retries': retries} |
| 47 | + return None |
| 48 | + |
| 49 | + |
| 50 | +def patch_scenario_with_accuracy(context, scenario, accuracy=0.9, retries=10): |
| 51 | + """Monkey-patches :func:`~behave.model.Scenario.run()` to execute multiple times and calculate the accuracy of the |
| 52 | + results. |
| 53 | +
|
| 54 | + This is helpful when the test is flaky due to unreliable test infrastructure or when the application under test is |
| 55 | + AI based and its responses may vary slightly. |
| 56 | +
|
| 57 | + :param context: behave context |
| 58 | + :param scenario: Scenario or ScenarioOutline to patch |
| 59 | + :param accuracy: Minimum accuracy required to consider the scenario as passed |
| 60 | + :param retries: Number of times the scenario will be executed |
| 61 | + """ |
| 62 | + def scenario_run_with_accuracy(context, scenario_run, scenario, *args, **kwargs): |
| 63 | + # Execute the scenario multiple times and count passed executions |
| 64 | + passed_executions = 0 |
| 65 | + for retry in range(1, retries+1): |
| 66 | + if not scenario_run(*args, **kwargs): |
| 67 | + passed_executions += 1 |
| 68 | + status = "PASSED" |
| 69 | + else: |
| 70 | + status = "FAILED" |
| 71 | + print(f"ACCURACY SCENARIO {status}: retry {retry}/{retries}") |
| 72 | + context.logger.info(f"Accuracy scenario {status} (retry {retry}/{retries})") |
| 73 | + |
| 74 | + # Calculate scenario accuracy |
| 75 | + scenario_accuracy = passed_executions / retries |
| 76 | + has_passed = scenario_accuracy >= accuracy |
| 77 | + final_status = 'PASSED' if has_passed else 'FAILED' |
| 78 | + print(f"\nACCURACY SCENARIO {final_status}: {retries} retries, accuracy {scenario_accuracy} >= {accuracy}") |
| 79 | + final_message = (f"Accuracy scenario {final_status} after {retries} retries with" |
| 80 | + f" accuracy {scenario_accuracy} >= {accuracy}") |
| 81 | + |
| 82 | + # Set final scenario status |
| 83 | + if has_passed: |
| 84 | + context.logger.info(final_message) |
| 85 | + scenario.set_status(Status.passed) |
| 86 | + else: |
| 87 | + context.logger.error(final_message) |
| 88 | + scenario.set_status(Status.failed) |
| 89 | + return not has_passed # Run method returns true when failed |
| 90 | + |
| 91 | + scenario_run = scenario.run |
| 92 | + scenario.run = functools.partial(scenario_run_with_accuracy, context, scenario_run, scenario) |
| 93 | + |
| 94 | + |
| 95 | +def patch_scenario_from_tags(context, scenario): |
| 96 | + """Patch scenario with accuracy method when accuracy tags are present in scenario. |
| 97 | +
|
| 98 | + :param context: behave context |
| 99 | + :param scenario: behave scenario |
| 100 | + """ |
| 101 | + accuracy_data = get_accuracy_and_retries_from_tags(scenario.effective_tags) |
| 102 | + if accuracy_data: |
| 103 | + patch_scenario_with_accuracy(context, scenario, accuracy=accuracy_data['accuracy'], |
| 104 | + retries=accuracy_data['retries']) |
| 105 | + |
| 106 | + |
| 107 | +def patch_feature_scenarios_with_accuracy(context, feature): |
| 108 | + """Patch feature scenarios with accuracy method when accuracy tags are present in scenarios. |
| 109 | +
|
| 110 | + :param context: behave context |
| 111 | + :param feature: behave feature |
| 112 | + """ |
| 113 | + try: |
| 114 | + for scenario in feature.scenarios: |
| 115 | + if isinstance(scenario, ScenarioOutline): |
| 116 | + for outline_scenario in scenario.scenarios: |
| 117 | + patch_scenario_from_tags(context, outline_scenario) |
| 118 | + else: |
| 119 | + patch_scenario_from_tags(context, scenario) |
| 120 | + except Exception as e: |
| 121 | + # Log error but do not fail the execution to avoid errors in before feature method |
| 122 | + context.logger.error(f"Error applying accuracy policy: {e}") |
0 commit comments