Merge pull request #22 from applitools/feature-add-get_all_test_results

Feature add get all test results

Author: eirenik0

eyes_common/applitools/common/__init__.py

@@ -29,7 +29,7 @@
 from .match_window_data import MatchWindowData, Options
 from .metadata import AppEnvironment, RunningSession, SessionStartInfo
 from .server import FailureReports, SessionType
-from .test_results import TestResults
+from .test_results import TestResults, TestResultSummary
 from .visual_grid import (
     ChromeEmulationInfo,
     DeviceName,

eyes_common/applitools/common/test_results.py

@@ -9,13 +9,13 @@
 from .match import ImageMatchSettings
 
 if typing.TYPE_CHECKING:
-    from typing import Text, Any, Dict, Optional
+    from typing import Text, Any, Dict, Optional, List
 
     # TODO: Implement objects
     SessionUrls = Dict[Any, Any]
     StepInfo = Dict[Any, Any]
 
-__all__ = ("TestResults",)
+__all__ = ("TestResults", "TestResultSummary")
 
 
 class TestResultsStatus(Enum):
@@ -98,3 +98,30 @@ def __str__(self):
         origin_str = super(TestResults, self).__str__()
         preamble = "New test" if self.is_new else "Existing test"
         return "{} [{}]".format(preamble, origin_str)
+
+
+@attr.s
+class TestResultSummary(object):
+    all_results = attr.ib()  # type: List[TestResults]
+    exceptions = attr.ib(default=0)
+
+    passed = attr.ib(init=False, default=0)
+    unresolved = attr.ib(init=False, default=0)
+    failed = attr.ib(init=False, default=0)
+    mismatches = attr.ib(init=False, default=0)
+    missing = attr.ib(init=False, default=0)
+    matches = attr.ib(init=False, default=0)
+
+    def __attrs_post_init__(self):
+        for result in self.all_results:
+            if result is None:
+                continue
+            if result.is_failed:
+                self.failed += 1
+            elif result.is_passed:
+                self.passed += 1
+            elif result.is_unresolved:
+                self.unresolved += 1
+            self.matches += result.matches
+            self.missing += result.missing
+            self.mismatches += result.mismatches

eyes_selenium/applitools/selenium/visual_grid/vg_task.py

@@ -3,10 +3,10 @@
 
 import attr
 
-from applitools.common import logger
+from applitools.common import TestResults, logger
 
 if typing.TYPE_CHECKING:
-    from typing import Callable, Text
+    from typing import Callable, Text, Optional
 
 
 @attr.s(hash=True)
@@ -35,7 +35,7 @@ def on_task_completed(self, code):
         return self
 
     def __call__(self):
-        # type: () -> None
+        # type: () -> Optional[TestResults]
         logger.debug("%s called %s" % (self.__class__.__name__, self.name))
         res = None
         try:

eyes_selenium/applitools/selenium/visual_grid/visual_grid_eyes.py

@@ -69,14 +69,14 @@ class VisualGridEyes(object):
     _is_opened = False
     _driver = None
     rendering_info = None
-    test_list = []  # type: List[RunningTest]
 
     def __init__(self, runner, config):
         # type: (VisualGridRunner, Eyes)-> None
-        self._elements = []
         self._config_provider = config
+        self._elements = []
         argument_guard.not_none(runner)
         self.vg_manager = runner
+        self.test_list = []  # type: List[RunningTest]
 
     @property
     def is_opened(self):
@@ -185,56 +185,21 @@ def close_async(self):
             test.close()
 
     def close(self, raise_ex=True):  # noqa
-        # type: (Optional[bool]) -> List[TestResults]
+        # type: (Optional[bool]) -> Optional[TestResults]
         if self.configuration.is_disabled:
             logger.debug("close(): ignored (disabled)")
-            return []
+            return TestResults()
         if not self.test_list:
-            return []
+            return TestResults()
         logger.debug("VisualGridEyes.close()\n\t test_list %s" % self.test_list)
         self.close_async()
-
-        while True:
-            completed_states = [
-                test.state for test in self.test_list if test.state == "completed"
-            ]
-            if len(completed_states) == len(self.test_list):
-                break
-            sleep(0.5)
+        self.vg_manager.process_test_list(self.test_list, raise_ex)
         self._is_opened = False
-        self.vg_manager.stop()
-        logger.close()
 
-        for test in self.test_list:
-            if test.pending_exceptions:
-                raise EyesError(
-                    "During test execution above exception raised. \n {}".join(
-                        test.pending_exceptions
-                    )
-                )
-
-        if raise_ex:
-            for test in self.test_list:
-                results = test.test_result
-                msg = "Test '{}' of '{}'. \n\tSee details at: {}".format(
-                    results.name, results.app_name, results.url
-                )
-                if results.is_unresolved and not results.is_new:
-                    raise DiffsFoundError(msg, results)
-                if results.is_new:
-                    raise NewTestError(msg, results)
-                if results.is_failed:
-                    raise TestFailedError(msg, results)
-
-        failed_results = [
-            test.test_result
-            for test in self.test_list
-            if test.test_result.is_unresolved or test.test_result.is_failed
-        ]
-        return failed_results
-
-    def get_all_test_results(self):
-        return [test.test_result for test in self.test_list]
+        test_results = [t.test_result for t in self.test_list if t.test_result]
+        if not test_results:
+            return TestResults()
+        return test_results[0]
 
     def _create_vgeyes_connector(self, b_info):
         # type: (RenderBrowserInfo) -> EyesConnector

eyes_selenium/applitools/selenium/visual_grid/visual_grid_runner.py

@@ -1,13 +1,20 @@
-import sys
 import concurrent
 import itertools
 import operator
+import sys
 import threading
 import typing
 from concurrent.futures import ThreadPoolExecutor
 from time import sleep
 
-from applitools.common import logger
+from applitools.common import (
+    DiffsFoundError,
+    EyesError,
+    NewTestError,
+    TestFailedError,
+    TestResultSummary,
+    logger,
+)
 from applitools.selenium.visual_grid.resource_cache import ResourceCache
 
 if typing.TYPE_CHECKING:
@@ -87,18 +94,49 @@ def stop(self):
         self.executor.shutdown()
         self.thread.join()
 
-    # def get_all_test_results(self):
-    #     # type: () -> List[TestResults]
-    #     while not any(e.is_opened for e in self.all_eyes):
-    #         sleep(0.5)
-    #     return list(
-    #         itertools.chain.from_iterable(
-    #             test.test_result
-    #             for e in self.all_eyes
-    #             for test in e.test_list
-    #             if e.test_list
-    #         )
-    #     )
+    def process_test_list(self, test_list, raise_ex):
+        while True:
+            completed_states = [
+                test.state for test in test_list if test.state == "completed"
+            ]
+            if len(completed_states) == len(test_list):
+                break
+            sleep(0.5)
+        self.stop()
+        logger.close()
+
+        for test in test_list:
+            if test.pending_exceptions:
+                raise EyesError(
+                    "During test execution above exception raised. \n {}".join(
+                        test.pending_exceptions
+                    )
+                )
+        if raise_ex:
+            for test in test_list:
+                results = test.test_result
+                msg = "Test '{}' of '{}'. \n\tSee details at: {}".format(
+                    results.name, results.app_name, results.url
+                )
+                if results.is_unresolved and not results.is_new:
+                    raise DiffsFoundError(msg, results)
+                if results.is_new:
+                    raise NewTestError(msg, results)
+                if results.is_failed:
+                    raise TestFailedError(msg, results)
+        return test_list
+
+    def get_all_test_results(self, raise_ex=True):
+        # type: (bool) -> TestResultSummary
+        while not any(e.is_opened for e in self.all_eyes):
+            sleep(0.5)
+        test_list = self.process_test_list(
+            [test for e in self.all_eyes for test in e.test_list], raise_ex
+        )
+        for e in self.all_eyes:
+            e._is_opened = False
+        exceptions = [exp for t in test_list for exp in t.pending_exceptions]
+        return TestResultSummary([t.test_result for t in test_list], len(exceptions))
 
     @property
     def all_running_tests(self):

tests/functional/eyes_selenium/visual_grid/test_vg_manager.py

@@ -0,0 +1,21 @@
+from applitools.selenium import BrowserType, Eyes
+
+
+def test_get_all_test_results(vg_runner, driver):
+    eyes1 = Eyes(vg_runner)
+    eyes2 = Eyes(vg_runner)
+    eyes1.configuration.add_browser(800, 600, BrowserType.CHROME)
+    eyes2.configuration.add_browser(700, 500, BrowserType.FIREFOX)
+
+    driver.get("https://demo.applitools.com")
+    eyes1.open(driver, "Testing1", "TestClose1")
+    eyes2.open(driver, "Testing2", "TestClose2")
+
+    eyes1.check_window()
+    eyes2.check_window()
+
+    eyes1.close_async()
+    eyes2.close_async()
+
+    results = vg_runner.get_all_test_results(False)
+    print(results)