Add support for the "selenium-wire" library

Author: mdmintz

README.md

@@ -565,7 +565,8 @@ pytest my_first_test.py --pdb
 --maximize  # (Start tests with the browser window maximized.)
 --screenshot  # (Save a screenshot at the end of each test.)
 --visual-baseline  # (Set the visual baseline for Visual/Layout tests.)
---external-pdf  # (Set Chrome "plugins.always_open_pdf_externally": True.)
+--wire  # (Use selenium-wire's webdriver for replacing selenium webdriver.)
+--external-pdf  # (Set Chromium "plugins.always_open_pdf_externally":True.)
 --timeout-multiplier=MULTIPLIER  # (Multiplies the default timeout values.)
 --list-fail-page  # (After each failing test, list the URL of the failure.)
 ```

examples/raw_parameter_script.py

@@ -74,6 +74,7 @@
     sb._reuse_session = False
     sb._crumbs = False
     sb._final_debug = False
+    sb.use_wire = False
     sb.visual_baseline = False
     sb.window_size = None
     sb.maximize_option = False

help_docs/customizing_test_runs.md

@@ -179,7 +179,8 @@ pytest my_first_test.py --settings-file=custom_settings.py
 --maximize  # (Start tests with the browser window maximized.)
 --screenshot  # (Save a screenshot at the end of each test.)
 --visual-baseline  # (Set the visual baseline for Visual/Layout tests.)
---external-pdf  # (Set Chrome "plugins.always_open_pdf_externally": True.)
+--wire  # (Use selenium-wire's webdriver for replacing selenium webdriver.)
+--external-pdf  # (Set Chromium "plugins.always_open_pdf_externally":True.)
 --timeout-multiplier=MULTIPLIER  # (Multiplies the default timeout values.)
 --list-fail-page  # (After each failing test, list the URL of the failure.)
 ```

help_docs/features_list.md

@@ -24,6 +24,7 @@
 * Can run tests with a customized browser user agent. (``--agent=USER_AGENT_STRING``)
 * Can set a Chromium User Data Directory/Profile to load. (``--user-data-dir=DIR``)
 * Can avoid detection by sites that try to block Selenium. (``--undetected``/``--uc``)
+* Can integrate with [selenium-wire](https://github.com/wkeeling/selenium-wire) for inspecting browser requests. (``--wire``)
 * Can load Chrome Extension ZIP files. (``--extension-zip=ZIP``)
 * Can load Chrome Extension folders. (``--extension-dir=DIR``)
 * Powerful [console scripts](https://github.com/seleniumbase/SeleniumBase/blob/master/seleniumbase/console_scripts/ReadMe.md). (Type **``seleniumbase``** or **``sbase``** to use.)

help_docs/method_summary.md

@@ -536,6 +536,10 @@ self.get_session_storage_items()
 
 ############
 
+self.set_wire_proxy(string)  # Requires "--wire"!
+
+############
+
 self.add_css_link(css_link)
 
 self.add_js_link(js_link)

help_docs/syntax_formats.md

@@ -73,7 +73,7 @@ class BaseTestCase(BaseCase):
         # <<< Run custom setUp() code for tests AFTER the super().setUp() >>>
 
     def tearDown(self):
-        self.save_teardown_screenshot()  # If test fails, or if "--screenshot"
+        self.save_teardown_screenshot()  # On failure or "--screenshot"
         if self.has_exception():
             # <<< Run custom code if the test failed. >>>
             pass
@@ -255,7 +255,7 @@ class OverrideDriverTest(BaseCase):
 
 (From <a href="https://github.com/seleniumbase/SeleniumBase/blob/master/examples/test_override_driver.py">examples/test_override_driver.py</a>)
 
-The above format can let you use [selenium-wire](https://github.com/wkeeling/selenium-wire) to intercept & inspect requests and responses during SeleniumBase tests. Here's how the ``selenium-wire`` integration may look:
+The above format lets you customize [selenium-wire](https://github.com/wkeeling/selenium-wire) for intercepting and inspecting requests and responses during SeleniumBase tests. Here's how a ``selenium-wire`` integration may look:
 
 ```python
 from seleniumbase import BaseCase
@@ -277,6 +277,8 @@ class WireTestCase(BaseCase):
             print(request.url)
 ```
 
+(NOTE: The ``selenium-wire`` integration is now included with ``seleniumbase``: Add ``--wire`` as a ``pytest`` command-line option to activate. If you need both ``--wire`` with ``--undetected`` together, you'll still need to override ``get_new_driver()``.)
+
 <a id="sb_sf_10"></a>
 <h3><img src="https://seleniumbase.github.io/img/logo3b.png" title="SeleniumBase" width="32" /> 10. Overriding the driver via "sb" fixture</h3>
 
@@ -297,7 +299,7 @@ def sb(request):
             super(BaseClass, self).setUp()
 
         def tearDown(self):
-            self.save_teardown_screenshot()
+            self.save_teardown_screenshot()  # On failure or "--screenshot"
             super(BaseClass, self).tearDown()
 
         def base_method(self):
@@ -352,7 +354,7 @@ def sb(request):
             super(BaseClass, self).setUp()
 
         def tearDown(self):
-            self.save_teardown_screenshot()
+            self.save_teardown_screenshot()  # On failure or "--screenshot"
             super(BaseClass, self).tearDown()
 
         def base_method(self):
@@ -390,6 +392,8 @@ class TestWire:
             print(request.url)
 ```
 
+(NOTE: The ``selenium-wire`` integration is now included with ``seleniumbase``: Add ``--wire`` as a ``pytest`` command-line option to activate. If you need both ``--wire`` with ``--undetected`` together, you'll still need to override ``get_new_driver()``.)
+
 <a id="sb_sf_11"></a>
 <h3><img src="https://seleniumbase.github.io/img/logo3b.png" title="SeleniumBase" width="32" /> 11. BaseCase with Chinese translations</h3>
 

seleniumbase/behave/behave_sb.py

@@ -89,6 +89,7 @@
 -D maximize  (Start tests with the browser window maximized.)
 -D screenshot  (Save a screenshot at the end of each test.)
 -D visual-baseline  (Set the visual baseline for Visual/Layout tests.)
+-D wire  (Use selenium-wire's webdriver for replacing selenium webdriver.)
 -D external-pdf (Set Chromium "plugins.always_open_pdf_externally": True.)
 -D timeout-multiplier=MULTIPLIER  (Multiplies the default timeout values.)
 """
@@ -179,6 +180,7 @@ def get_configured_sb(context):
     sb._crumbs = False
     sb._disable_beforeunload = False
     sb.visual_baseline = False
+    sb.use_wire = False
     sb.window_size = None
     sb.maximize_option = False
     sb.is_context_manager = False
@@ -540,6 +542,10 @@ def get_configured_sb(context):
         if low_key in ["visual-baseline", "visual_baseline"]:
             sb.visual_baseline = True
             continue
+        # Handle: -D wire
+        if low_key == "wire":
+            sb.use_wire = True
+            continue
         # Handle: -D window-size=Width,Height / window_size=Width,Height
         if low_key in ["window-size", "window_size"]:
             window_size = userdata[key]

seleniumbase/core/browser_launcher.py

@@ -16,6 +16,7 @@
 from seleniumbase.core import download_helper
 from seleniumbase.core import proxy_helper
 from seleniumbase.fixtures import constants
+from seleniumbase.fixtures import shared_utils
 from seleniumbase import drivers  # webdriver storage folder for SeleniumBase
 from seleniumbase import extensions  # browser extensions storage folder
 
@@ -372,6 +373,7 @@ def _set_chrome_options(
     extension_zip,
     extension_dir,
     page_load_strategy,
+    use_wire,
     external_pdf,
     servername,
     mobile_emulator,
@@ -918,6 +920,7 @@ def get_driver(
     extension_zip=None,
     extension_dir=None,
     page_load_strategy=None,
+    use_wire=False,
     external_pdf=False,
     test_id=None,
     mobile_emulator=False,
@@ -1083,6 +1086,7 @@ def get_driver(
             extension_zip,
             extension_dir,
             page_load_strategy,
+            use_wire,
             external_pdf,
             test_id,
             mobile_emulator,
@@ -1130,6 +1134,7 @@ def get_driver(
             extension_zip,
             extension_dir,
             page_load_strategy,
+            use_wire,
             external_pdf,
             mobile_emulator,
             device_width,
@@ -1181,13 +1186,29 @@ def get_remote_driver(
     extension_zip,
     extension_dir,
     page_load_strategy,
+    use_wire,
     external_pdf,
     test_id,
     mobile_emulator,
     device_width,
     device_height,
     device_pixel_ratio,
 ):
+    if use_wire and selenium4_or_newer:
+        driver_fixing_lock = fasteners.InterProcessLock(
+            constants.MultiBrowser.DRIVER_FIXING_LOCK
+        )
+        with driver_fixing_lock:  # Prevent multi-processes mode issues
+            try:
+                from seleniumwire import webdriver
+            except Exception:
+                shared_utils.pip_install(
+                    "selenium-wire", version=constants.SeleniumWire.VER
+                )
+                from seleniumwire import webdriver
+    else:
+        from selenium import webdriver
+
     # Construct the address for connecting to a Selenium Grid
     if servername.startswith("https://"):
         protocol = "https"
@@ -1280,6 +1301,7 @@ def get_remote_driver(
             extension_zip,
             extension_dir,
             page_load_strategy,
+            use_wire,
             external_pdf,
             servername,
             mobile_emulator,
@@ -1510,6 +1532,7 @@ def get_remote_driver(
             extension_zip,
             extension_dir,
             page_load_strategy,
+            use_wire,
             external_pdf,
             servername,
             mobile_emulator,
@@ -1708,6 +1731,7 @@ def get_local_driver(
     extension_zip,
     extension_dir,
     page_load_strategy,
+    use_wire,
     external_pdf,
     mobile_emulator,
     device_width,
@@ -1719,6 +1743,20 @@ def get_local_driver(
     Can also be used to spin up additional browsers for the same test.
     """
     downloads_path = DOWNLOADS_FOLDER
+    if use_wire and selenium4_or_newer:
+        driver_fixing_lock = fasteners.InterProcessLock(
+            constants.MultiBrowser.DRIVER_FIXING_LOCK
+        )
+        with driver_fixing_lock:  # Prevent multi-processes mode issues
+            try:
+                from seleniumwire import webdriver
+            except Exception:
+                shared_utils.pip_install(
+                    "selenium-wire", version=constants.SeleniumWire.VER
+                )
+                from seleniumwire import webdriver
+    else:
+        from selenium import webdriver
 
     if browser_name == constants.Browser.FIREFOX:
         firefox_options = _set_firefox_options(
@@ -2308,6 +2346,7 @@ def get_local_driver(
                 extension_zip,
                 extension_dir,
                 page_load_strategy,
+                use_wire,
                 external_pdf,
                 servername,
                 mobile_emulator,
@@ -2372,6 +2411,7 @@ def get_local_driver(
                 extension_zip,
                 extension_dir,
                 page_load_strategy,
+                use_wire,
                 external_pdf,
                 servername,
                 mobile_emulator,
@@ -2754,6 +2794,7 @@ def get_local_driver(
                         extension_zip,
                         extension_dir,
                         page_load_strategy,
+                        use_wire,
                         external_pdf,
                         servername,
                         mobile_emulator,

seleniumbase/fixtures/base_case.py

@@ -3201,6 +3201,7 @@ def get_new_driver(
         extension_zip=None,
         extension_dir=None,
         page_load_strategy=None,
+        use_wire=None,
         external_pdf=None,
         is_mobile=None,
         d_width=None,
@@ -3251,6 +3252,7 @@ def get_new_driver(
         extension_zip - A Chrome Extension ZIP file to use (Chrome-only)
         extension_dir - A Chrome Extension folder to use (Chrome-only)
         page_load_strategy - the option to change pageLoadStrategy (Chrome)
+        use_wire - Use selenium-wire webdriver instead of the selenium one
         external_pdf - "plugins.always_open_pdf_externally": True. (Chrome)
         is_mobile - the option to use the mobile emulator (Chrome-only)
         d_width - the device width of the mobile emulator (Chrome-only)
@@ -3367,6 +3369,8 @@ def get_new_driver(
             extension_dir = self.extension_dir
         if page_load_strategy is None:
             page_load_strategy = self.page_load_strategy
+        if use_wire is None:
+            use_wire = self.use_wire
         if external_pdf is None:
             external_pdf = self.external_pdf
         test_id = self.__get_test_id()
@@ -3432,6 +3436,7 @@ def get_new_driver(
             extension_zip=extension_zip,
             extension_dir=extension_dir,
             page_load_strategy=page_load_strategy,
+            use_wire=use_wire,
             external_pdf=external_pdf,
             test_id=test_id,
             mobile_emulator=is_mobile,
@@ -7773,6 +7778,36 @@ def get_session_storage_items(self):
 
     ############
 
+    # Methods ONLY for the selenium-wire integration ("--wire")
+
+    def set_wire_proxy(self, string):
+        """Set a proxy server for selenium-wire mode ("--wire")
+        NOTE: This method ONLY works while using "--wire" mode!
+        Examples:
+            self.set_wire_proxy("SERVER:PORT")
+            self.set_wire_proxy("socks5://SERVER:PORT")
+            self.set_wire_proxy("USERNAME:PASSWORD@SERVER:PORT")
+        """
+        if not string:
+            self.driver.proxy = {}
+            return
+        the_http = "http"
+        the_https = "https"
+        if string.startswith("socks4://"):
+            the_http = "socks4"
+            the_https = "socks4"
+        elif string.startswith("socks5://"):
+            the_http = "socks5"
+            the_https = "socks5"
+        string = string.split("//")[-1]
+        self.driver.proxy = {
+            "http": "%s://%s" % (the_http, string),
+            "https": "%s://%s" % (the_https, string),
+            "no_proxy": "localhost,127.0.0.1",
+        }
+
+    ############
+
     # Duplicates (Avoids name confusion when migrating from other frameworks.)
 
     def open_url(self, url):
@@ -12804,6 +12839,7 @@ def setUp(self, masterqa_mode=False):
             self.extension_zip = sb_config.extension_zip
             self.extension_dir = sb_config.extension_dir
             self.page_load_strategy = sb_config.page_load_strategy
+            self.use_wire = sb_config.use_wire
             self.external_pdf = sb_config.external_pdf
             self._final_debug = sb_config.final_debug
             self.window_size = sb_config.window_size
@@ -13091,6 +13127,7 @@ def setUp(self, masterqa_mode=False):
                 extension_zip=self.extension_zip,
                 extension_dir=self.extension_dir,
                 page_load_strategy=self.page_load_strategy,
+                use_wire=self.use_wire,
                 external_pdf=self.external_pdf,
                 is_mobile=self.mobile_emulator,
                 d_width=self.__device_width,

seleniumbase/fixtures/constants.py

@@ -293,6 +293,11 @@ class Tether:
     )
 
 
+class SeleniumWire:
+    # The version installed if selenium-wire is not installed
+    VER = "5.1.0"
+
+
 class ValidBrowsers:
     valid_browsers = [
         "chrome",