Merge pull request #1545 from seleniumbase/performance-testing

Performance Testing decorators

Author: mdmintz

examples/example_logs/ReadMe.md

@@ -72,7 +72,7 @@ pytest test_suite.py --dashboard --html=report.html
 
 --------
 
-If viewing ``pytest-html`` reports in [Jenkins](https://www.jenkins.io/), you may need to [configure Jenkins settings](https://stackoverflow.com/a/46197356) for the HTML to render correctly. This is due to [Jenkins CSP changes](https://www.jenkins.io/doc/book/system-administration/security/configuring-content-security-policy/). That setting can be changed from ``Manage Jenkins`` > ``Script Console`` by running:
+If viewing ``pytest-html`` reports in [Jenkins](https://www.jenkins.io/), you may need to [configure Jenkins settings](https://stackoverflow.com/a/46197356) for the HTML to render correctly. This is due to [Jenkins CSP changes](https://www.jenkins.io/doc/book/security/configuring-content-security-policy/). That setting can be changed from ``Manage Jenkins`` > ``Script Console`` by running:
 
 ```
 System.setProperty("hudson.model.DirectoryBrowserSupport.CSP", "")

examples/performance_test.py

@@ -0,0 +1,52 @@
+"""
+Performance test example.
+
+Uses decorators.print_runtime(), which prints the runtime duration
+of a method or "with"-block after the method (or block) completes.
+Also raises an exception when exceeding the time "limit" if set.
+
+Arguments ->
+    description  # Optional - Shows description in print output.
+    limit  # Optional - Fail if the duration is above the limit.
+
+Method / Function example usage ->
+    from seleniumbase import decorators
+
+    @decorators.print_runtime("My Method")
+    def my_method():
+        # code ...
+        # code ...
+
+"with"-block example usage ->
+    from seleniumbase import decorators
+
+    with decorators.print_runtime("My Code Block"):
+        # code ...
+        # code ...
+"""
+from seleniumbase import BaseCase
+from seleniumbase import decorators
+
+
+class PerformanceClass(BaseCase):
+    @decorators.print_runtime("Open Swag Labs and Log In")
+    def login_to_swag_labs(self):
+        with decorators.print_runtime("Open Swag Labs"):
+            self.open("https://www.saucedemo.com")
+        self.type("#user-name", "standard_user")
+        self.type("#password", "secret_sauce\n")
+
+    def test_performance_of_swag_labs(self):
+        self.login_to_swag_labs()
+        self.assert_element("div.inventory_list")
+        self.assert_exact_text("PRODUCTS", "span.title")
+        with decorators.print_runtime("Add backpack and see cart"):
+            self.click('button[name*="backpack"]')
+            self.click("#shopping_cart_container a")
+            self.assert_text("Backpack", "div.cart_item")
+        with decorators.print_runtime("Remove backpack from cart"):
+            self.click('button:contains("Remove")')  # HTML innerText
+            self.assert_text_not_visible("Backpack", "div.cart_item")
+        with decorators.print_runtime("Log out from Swag Labs", 3):
+            self.js_click("a#logout_sidebar_link")
+            self.assert_element("div#login_button_container")

examples/time_limit_test.py

@@ -1,18 +1,27 @@
-""" This test fails on purpose to demonstrate the time-limit feature
-    for tests that run longer than the time limit specified (seconds).
-    The time-limit clock starts after the browser has fully launched,
-    which is after pytest starts it's own internal clock for tests.
-    Usage: (inside tests) =>  self.set_time_limit(SECONDS)
-    Usage: (command-line) =>  --time-limit=SECONDS """
-
 import pytest
 from seleniumbase import BaseCase
+from seleniumbase import decorators
 
 
 class TimeLimitTests(BaseCase):
     @pytest.mark.expected_failure
-    def test_time_limit_feature(self):
-        self.set_time_limit(5)  # Fail test if time exceeds 5 seconds
-        self.open("https://xkcd.com/1658/")
+    def test_runtime_limit_decorator(self):
+        """This test fails on purpose to show the runtime_limit() decorator
+        for code blocks that run longer than the time limit specified."""
+        print("\n(This test should fail)")
+        self.open("https://xkcd.com/1190")
+        with decorators.runtime_limit(0.5):
+            self.sleep(0.75)
+
+    @pytest.mark.expected_failure
+    def test_set_time_limit_method(self):
+        """This test fails on purpose to show the set_time_limit() method
+        for tests that run longer than the time limit specified (seconds).
+        The time-limit clock starts after the browser has fully launched,
+        which is after pytest starts it's own internal clock for tests.
+        Usage: (inside tests) =>  self.set_time_limit(SECONDS)
+        Usage: (command-line) =>  --time-limit=SECONDS"""
+        self.set_time_limit(2.5)  # Fail test if time exceeds 2.5 seconds
         print("\n(This test should fail)")
-        self.sleep(7)
+        self.open("https://xkcd.com/1658")
+        self.sleep(3)

mkdocs_build/requirements.txt

@@ -12,7 +12,7 @@ keyring==23.9.3
 pkginfo==1.8.3
 Jinja2==3.1.2
 click==8.1.3
-zipp==3.8.1
+zipp==3.9.0
 ghp-import==2.1.0
 readme-renderer==37.2
 pymdown-extensions==9.6

requirements.txt

@@ -44,6 +44,7 @@ urllib3==1.26.12
 requests==2.27.1;python_version<"3.6"
 requests==2.27.1;python_version>="3.6" and python_version<"3.7"
 requests==2.28.1;python_version>="3.7"
+requests-toolbelt==0.10.0
 nose==1.3.7
 sniffio==1.3.0;python_version>="3.7"
 h11==0.14.0;python_version>="3.7"

seleniumbase/__version__.py

@@ -1,2 +1,2 @@
 # seleniumbase package
-__version__ = "4.5.5"
+__version__ = "4.5.6"

seleniumbase/common/ReadMe.md

@@ -4,6 +4,10 @@
 
 #### Use these Python decorators with your test methods as needed:
 
+* @print_runtime(description=None, limit=None)
+
+* @runtime_limit(limit, description=None)
+
 * @retry_on_exception(tries=6, delay=1, backoff=2, max_delay=32)
 
 * @rate_limited(max_per_second)

seleniumbase/common/decorators.py

@@ -3,9 +3,118 @@
 import sys
 import time
 import warnings
+from contextlib import contextmanager
 from functools import wraps
 
 
+@contextmanager
+def print_runtime(description=None, limit=None):
+    """Print the runtime duration of a method or "with"-block after completion.
+    If limit, fail if the runtime duration exceeds the limit after completion.
+
+    Method / Function example usage ->
+        from seleniumbase import decorators
+
+        @decorators.print_runtime("My Method")
+        def my_method():
+            # code ...
+            # code ...
+
+    "with"-block example usage ->
+        from seleniumbase import decorators
+
+        with decorators.print_runtime("My Code Block"):
+            # code ...
+            # code ...
+    """
+    if not description:
+        description = "Code Block"
+    description = str(description)
+    if limit:
+        limit = float("%.2f" % limit)
+        if limit < 0.01:
+            limit = 0.01  # Minimum runtime limit
+    exception = None
+    start_time = time.time()
+    try:
+        yield
+    except Exception as e:
+        exception = e
+        raise
+    finally:
+        end_time = time.time()
+        run_time = end_time - start_time
+        # Print times with a statistically significant number of decimal places
+        if run_time < 0.0001:
+            print("  {%s} ran for %.7f seconds." % (description, run_time))
+        elif run_time < 0.001:
+            print("  {%s} ran for %.6f seconds." % (description, run_time))
+        elif run_time < 0.01:
+            print("  {%s} ran for %.5f seconds." % (description, run_time))
+        elif run_time < 0.1:
+            print("  {%s} ran for %.4f seconds." % (description, run_time))
+        elif run_time < 1:
+            print("  {%s} ran for %.3f seconds." % (description, run_time))
+        else:
+            print("  {%s} ran for %.2f seconds." % (description, run_time))
+        if limit and limit > 0 and run_time > limit:
+            message = (
+                "\n {%s} duration of %.2fs exceeded the time limit of %.2fs!"
+                % (description, run_time, limit)
+            )
+            if exception:
+                message = exception.msg + "\nAND " + message
+            raise Exception(message)
+
+
+@contextmanager
+def runtime_limit(limit, description=None):
+    """
+    Fail if the runtime duration of a method or "with"-block exceeds the limit.
+    (The failure won't occur until after the method or "with"-block completes.)
+
+    Method / Function example usage ->
+        from seleniumbase import decorators
+
+        @decorators.runtime_limit(4.5)
+        def my_method():
+            # code ...
+            # code ...
+
+    "with"-block example usage ->
+        from seleniumbase import decorators
+
+        with decorators.runtime_limit(32):
+            # code ...
+            # code ...
+    """
+    limit = float("%.2f" % limit)
+    if limit < 0.01:
+        limit = 0.01  # Minimum runtime limit
+    if not description:
+        description = "Code Block"
+    description = str(description)
+    exception = None
+    start_time = time.time()
+    try:
+        yield
+    except Exception as e:
+        exception = e
+        raise
+    finally:
+        end_time = time.time()
+        run_time = end_time - start_time
+        # Fail if the runtime of the code block exceeds the limit
+        if limit and limit > 0 and run_time > limit:
+            message = (
+                "\n {%s} duration of %.2fs exceeded the time limit of %.2fs!"
+                % (description, run_time, limit)
+            )
+            if exception:
+                message = exception.msg + "\nAND " + message
+            raise Exception(message)
+
+
 def retry_on_exception(tries=6, delay=1, backoff=2, max_delay=32):
     """
     Decorator for implementing exponential backoff for retrying on failures.

setup.py

@@ -169,6 +169,7 @@
         'urllib3==1.26.12',  # Stay in sync with "requests"
         'requests==2.27.1;python_version<"3.7"',
         'requests==2.28.1;python_version>="3.7"',
+        'requests-toolbelt==0.10.0',
         "nose==1.3.7",
         'sniffio==1.3.0;python_version>="3.7"',
         'h11==0.14.0;python_version>="3.7"',