fix: count skipped tests in accuracy executions

Author: rgonalo

CHANGELOG.rst

@@ -6,6 +6,8 @@ v3.6.1
 
 *Release date: In development*
 
+- Fix accuracy tag to exclude skipped executions from total executions count
+
 v3.6.0
 ------
 

toolium/utils/ai_utils/accuracy.py

@@ -89,7 +89,7 @@ def patch_scenario_with_accuracy(context, scenario, data_key_suffix, accuracy=0.
     """
     def scenario_run_with_accuracy(context, scenario_run, scenario, *args, **kwargs):
         # Execute the scenario multiple times and count passed executions
-        passed_executions = 0
+        passed_executions = skipped_executions = 0
         # Copy scenario steps to avoid modifications in each execution, especially when using behave variables
         # transformation, like map_param and replace_param methods
         orig_steps = deepcopy(scenario.steps)
@@ -99,30 +99,64 @@ def scenario_run_with_accuracy(context, scenario_run, scenario, *args, **kwargs)
             # Restore original steps before each execution
             scenario.steps = deepcopy(orig_steps)
             if not scenario_run(*args, **kwargs):
-                passed_executions += 1
-                status = "PASSED"
+                if scenario.status == Status.skipped:
+                    skipped_executions += 1
+                    status = "SKIPPED"
+                else:
+                    passed_executions += 1
+                    status = "PASSED"
             else:
                 status = "FAILED"
             print(f"ACCURACY SCENARIO {status}: execution {execution+1}/{executions}")
             context.logger.info(f"Accuracy scenario execution {status} ({execution+1}/{executions})")
 
-        # Calculate scenario accuracy
-        scenario_accuracy = passed_executions / executions
-        has_passed = scenario_accuracy >= accuracy
-        final_status = 'PASSED' if has_passed else 'FAILED'
-        print(f"\nACCURACY SCENARIO {final_status}: {executions} executions,"
-              f" accuracy {scenario_accuracy} >= {accuracy}")
-        final_message = (f"Accuracy scenario {final_status} after {executions} executions with"
-                         f" accuracy {scenario_accuracy} >= {accuracy}")
-
-        # Set final scenario status
-        if has_passed:
-            context.logger.info(final_message)
-            scenario.set_status(Status.passed)
+
+        if executions == skipped_executions:
+            run_response = False  # Run method returns true only when failed
+            context.logger.info("All accuracy scenario executions are skipped")
         else:
-            context.logger.error(final_message)
-            scenario.set_status(Status.failed)
-        return not has_passed  # Run method returns true when failed
+            # Calculate scenario accuracy
+            scenario_accuracy = passed_executions / (executions - skipped_executions)
+            has_passed = scenario_accuracy >= accuracy
+            run_response = not has_passed  # Run method returns true only when failed
+            final_status = 'PASSED' if has_passed else 'FAILED'
+            print(f"\nACCURACY SCENARIO {final_status}: {executions} executions,"
+                  f" accuracy {scenario_accuracy} >= {accuracy}")
+            final_message = (f"Accuracy scenario {final_status} after {executions} executions with"
+                             f" accuracy {scenario_accuracy} >= {accuracy}"
+                             f" ({passed_executions} passed, {skipped_executions} skipped,"
+                             f" {executions - passed_executions - skipped_executions} failed)")
+
+            # Set final scenario status
+            if has_passed:
+                context.logger.info(final_message)
+                scenario.set_status(Status.passed)
+            else:
+                context.logger.error(final_message)
+                scenario.set_status(Status.failed)
+                scenario.exception = AssertionError(final_message)
+
+        # Clean accuracy execution data from context
+        context.storage.pop("accuracy_execution_data", None)
+        context.storage.pop("accuracy_execution_index", None)
+
+        # Execute Allure after_scenario hook if Allure is being used
+        if hasattr(context, 'allure'):
+            if scenario.status == Status.passed:
+                # Change failed steps to avoid marking scenario as failed in Allure when accuracy is met
+                for step in scenario.all_steps:
+                    if step.status == Status.failed:
+                        step.set_status(Status.passed)
+                        step.exception = None
+            elif scenario.status == Status.failed:
+                latest_step = list(scenario.all_steps)[-1]
+                latest_step.set_status(Status.failed)
+                latest_step.exception = AssertionError(final_message)
+            context.allure.listener.behave_config = context
+            context.allure.listener.behave_config.show_skipped = True
+            context.allure.after_scenario(context, scenario)
+
+        return run_response
 
     scenario_run = scenario.run
     scenario.run = functools.partial(scenario_run_with_accuracy, context, scenario_run, scenario)