Add support for undetected-chromedriver as a subprocess

Author: mdmintz

help_docs/customizing_test_runs.md

@@ -162,7 +162,8 @@ pytest my_first_test.py --settings-file=custom_settings.py
 --enable-ws  # (Enable Web Security on Chromium-based browsers.)
 --enable-sync  # (Enable "Chrome Sync" on websites.)
 --use-auto-ext  # (Use Chrome's automation extension.)
---undetected | --uc  # (Use undetected-chromedriver to evade bot-detection.)
+--uc | --undetected  # (Use undetected-chromedriver to evade bot-detection.)
+--uc-sub | --uc-subprocess  # (Use undetected-chromedriver as a subprocess.)
 --remote-debug  # (Enable Chrome's Remote Debugger on http://localhost:9222)
 --final-debug  # (Enter Debug Mode after each test ends. Don't use with CI!)
 --dashboard  # (Enable the SeleniumBase Dashboard. Saved at: dashboard.html)

seleniumbase/behave/behave_sb.py

@@ -72,7 +72,8 @@
 -D enable-ws  (Enable Web Security on Chromium-based browsers.)
 -D enable-sync  (Enable "Chrome Sync".)
 -D use-auto-ext  (Use Chrome's automation extension.)
--D undetected | -D uc  (Use undetected-chromedriver to evade bot-detection)
+-D uc | -D undetected  (Use undetected-chromedriver to evade bot-detection)
+-D uc-sub | -D uc-subprocess  (Use undetected-chromedriver as a subprocess)
 -D remote-debug  (Enable Chrome's Remote Debugger on http://localhost:9222)
 -D dashboard  (Enable the SeleniumBase Dashboard. Saved at: dashboard.html)
 -D dash-title=STRING  (Set the title shown for the generated dashboard.)
@@ -169,6 +170,7 @@ def get_configured_sb(context):
     sb.enable_sync = False
     sb.use_auto_ext = False
     sb.undetectable = False
+    sb.uc_subprocess = False
     sb.no_sandbox = False
     sb.disable_gpu = False
     sb._multithreaded = False
@@ -497,10 +499,15 @@ def get_configured_sb(context):
         if low_key in ["use-auto-ext", "use_auto_ext", "auto-ext"]:
             sb.use_auto_ext = True
             continue
-        # Handle: -D use-auto-ext / use_auto_ext / auto-ext
+        # Handle: -D undetected / undetectable / uc
         if low_key in ["undetected", "undetectable", "uc"]:
             sb.undetectable = True
             continue
+        # Handle: -D uc-subprocess / uc_subprocess / uc-sub
+        if low_key in ["uc-subprocess", "uc_subprocess", "uc-sub"]:
+            sb.uc_subprocess = True
+            sb.undetectable = True
+            continue
         # Handle: -D no-sandbox / no_sandbox
         if low_key in ["no-sandbox", "no_sandbox"]:
             sb.no_sandbox = True
@@ -748,18 +755,17 @@ def get_configured_sb(context):
     if sb.recorder_ext and sb.headless:
         sb.headless = False
         sb.headless2 = True
-    if sb.browser not in ["chrome", "edge"]:
+    if sb.headless2 and sb.browser == "firefox":
+        sb.headless2 = False  # Only for Chromium browsers
+        sb.headless = True  # Firefox has regular headless
+    elif sb.browser not in ["chrome", "edge"]:
         sb.headless2 = False  # Only for Chromium browsers
     # Recorder Mode only supports Chromium browsers.
     if sb.recorder_ext and (sb.browser not in ["chrome", "edge"]):
         raise Exception(
             "\n\n  Recorder Mode ONLY supports Chrome and Edge!"
             '\n  (Your browser choice was: "%s")\n' % sb.browser
         )
-    # Recorder Mode can still optimize scripts in --headless2 mode.
-    if sb.recorder_mode and sb.headless:
-        sb.headless = False
-        sb.headless2 = True
     # The Xvfb virtual display server is for Linux OS Only.
     if sb.xvfb and "linux" not in sys.platform:
         sb.xvfb = False
@@ -778,6 +784,10 @@ def get_configured_sb(context):
             'or by calling the new "-D headless2".)'
         )
         sb.headless = True
+    # Recorder Mode can still optimize scripts in --headless2 mode.
+    if sb.recorder_mode and sb.headless:
+        sb.headless = False
+        sb.headless2 = True
     if not sb.headless and not sb.headless2:
         sb.headed = True
     if sb.servername != "localhost":

seleniumbase/core/browser_launcher.py

@@ -354,6 +354,7 @@ def _set_chrome_options(
     enable_sync,
     use_auto_ext,
     undetectable,
+    uc_subprocess,
     no_sandbox,
     disable_gpu,
     headless2,
@@ -895,6 +896,7 @@ def get_driver(
     enable_sync=False,
     use_auto_ext=False,
     undetectable=False,
+    uc_subprocess=False,
     no_sandbox=False,
     disable_gpu=False,
     headless2=False,
@@ -1046,6 +1048,7 @@ def get_driver(
             enable_sync,
             use_auto_ext,
             undetectable,
+            uc_subprocess,
             no_sandbox,
             disable_gpu,
             headless2,
@@ -1091,6 +1094,7 @@ def get_driver(
             enable_sync,
             use_auto_ext,
             undetectable,
+            uc_subprocess,
             no_sandbox,
             disable_gpu,
             headless2,
@@ -1140,6 +1144,7 @@ def get_remote_driver(
     enable_sync,
     use_auto_ext,
     undetectable,
+    uc_subprocess,
     no_sandbox,
     disable_gpu,
     headless2,
@@ -1239,6 +1244,7 @@ def get_remote_driver(
             enable_sync,
             use_auto_ext,
             undetectable,
+            uc_subprocess,
             no_sandbox,
             disable_gpu,
             headless2,
@@ -1467,6 +1473,7 @@ def get_remote_driver(
             enable_sync,
             use_auto_ext,
             undetectable,
+            uc_subprocess,
             no_sandbox,
             disable_gpu,
             headless2,
@@ -1661,6 +1668,7 @@ def get_local_driver(
     enable_sync,
     use_auto_ext,
     undetectable,
+    uc_subprocess,
     no_sandbox,
     disable_gpu,
     headless2,
@@ -2260,6 +2268,7 @@ def get_local_driver(
                 enable_sync,
                 use_auto_ext,
                 undetectable,
+                uc_subprocess,
                 no_sandbox,
                 disable_gpu,
                 headless2,
@@ -2322,6 +2331,7 @@ def get_local_driver(
                 enable_sync,
                 use_auto_ext,
                 undetectable,
+                uc_subprocess,
                 no_sandbox,
                 disable_gpu,
                 headless2,
@@ -2592,6 +2602,7 @@ def get_local_driver(
                                             driver_executable_path=uc_path,
                                             headless=False,  # Xvfb needed!
                                             version_main=uc_chrome_version,
+                                            use_subprocess=uc_subprocess,
                                         )
                                     except URLError as e:
                                         if (
@@ -2609,6 +2620,7 @@ def get_local_driver(
                                                 driver_executable_path=uc_path,
                                                 headless=False,  # Xvfb needed!
                                                 version_main=uc_chrome_version,
+                                                use_subprocess=uc_subprocess,
                                             )
                                         else:
                                             raise
@@ -2700,6 +2712,7 @@ def get_local_driver(
                         enable_sync,
                         use_auto_ext,
                         undetectable,
+                        uc_subprocess,
                         no_sandbox,
                         disable_gpu,
                         headless2,

seleniumbase/fixtures/base_case.py

@@ -3181,6 +3181,7 @@ def get_new_driver(
         enable_sync=None,
         use_auto_ext=None,
         undetectable=None,
+        uc_subprocess=None,
         no_sandbox=None,
         disable_gpu=None,
         headless2=None,
@@ -3229,6 +3230,7 @@ def get_new_driver(
         enable_sync - the option to enable the Chrome Sync feature (Chrome)
         use_auto_ext - the option to enable Chrome's Automation Extension
         undetectable - the option to use an undetectable chromedriver
+        uc_subprocess - use the undetectable chromedriver as a subprocess
         no_sandbox - the option to enable the "No-Sandbox" feature (Chrome)
         disable_gpu - the option to enable Chrome's "Disable GPU" feature
         headless2 - the option to use the newer headless mode (Chromium)
@@ -3323,6 +3325,8 @@ def get_new_driver(
             use_auto_ext = self.use_auto_ext
         if undetectable is None:
             undetectable = self.undetectable
+        if uc_subprocess is None:
+            uc_subprocess = self.uc_subprocess
         if no_sandbox is None:
             no_sandbox = self.no_sandbox
         if disable_gpu is None:
@@ -3404,6 +3408,7 @@ def get_new_driver(
             enable_sync=enable_sync,
             use_auto_ext=use_auto_ext,
             undetectable=undetectable,
+            uc_subprocess=uc_subprocess,
             no_sandbox=no_sandbox,
             disable_gpu=disable_gpu,
             headless2=headless2,
@@ -12545,6 +12550,7 @@ def __activate_virtual_display_as_needed(self):
                 self.display = Display(visible=0, size=(width, height))
                 self.display.start()
                 self.headless_active = True
+                sb_config.headless_active = True
             except Exception:
                 pass
 
@@ -12630,6 +12636,7 @@ def setUp(self, masterqa_mode=False):
             self.with_selenium = sb_config.with_selenium  # Should be True
             self.headless = sb_config.headless
             self.headless_active = False
+            sb_config.headless_active = False
             self.headed = sb_config.headed
             self.xvfb = sb_config.xvfb
             self.locale_code = sb_config.locale_code
@@ -12687,6 +12694,7 @@ def setUp(self, masterqa_mode=False):
             self.enable_sync = sb_config.enable_sync
             self.use_auto_ext = sb_config.use_auto_ext
             self.undetectable = sb_config.undetectable
+            self.uc_subprocess = sb_config.uc_subprocess
             self.no_sandbox = sb_config.no_sandbox
             self.disable_gpu = sb_config.disable_gpu
             self.headless2 = sb_config.headless2
@@ -12978,6 +12986,7 @@ def setUp(self, masterqa_mode=False):
                 enable_sync=self.enable_sync,
                 use_auto_ext=self.use_auto_ext,
                 undetectable=self.undetectable,
+                uc_subprocess=self.uc_subprocess,
                 no_sandbox=self.no_sandbox,
                 disable_gpu=self.disable_gpu,
                 headless2=self.headless2,

seleniumbase/plugins/pytest_plugin.py

@@ -90,7 +90,8 @@ def pytest_addoption(parser):
     --enable-ws  (Enable Web Security on Chromium-based browsers.)
     --enable-sync  (Enable "Chrome Sync" on websites.)
     --use-auto-ext  (Use Chrome's automation extension.)
-    --undetected | --uc  (Use undetected-chromedriver to evade bot-detection.)
+    --uc | --undetected  (Use undetected-chromedriver to evade bot-detection.)
+    --uc-sub | --uc-subprocess  (Use undetected-chromedriver as a subprocess.)
     --remote-debug  (Enable Chrome's Remote Debugger on http://localhost:9222)
     --final-debug  (Enter Debug Mode after each test ends. Don't use with CI!)
     --dashboard  (Enable the SeleniumBase Dashboard. Saved at: dashboard.html)
@@ -910,6 +911,17 @@ def pytest_addoption(parser):
                 to websites that use anti-bot services to block
                 automation tools from navigating them freely.""",
     )
+    parser.addoption(
+        "--uc_subprocess",
+        "--uc-subprocess",
+        "--uc-sub",  # undetected-chromedriver subprocess mode
+        action="store_true",
+        dest="uc_subprocess",
+        default=False,
+        help="""Use undetectable-chromedriver as a subprocess,
+                which can help avoid issues that might result.
+                It may reduce UC's ability to avoid detection.""",
+    )
     parser.addoption(
         "--no_sandbox",
         "--no-sandbox",
@@ -1252,6 +1264,9 @@ def pytest_addoption(parser):
             "--undetected" in sys_argv
             or "--undetectable" in sys_argv
             or "--uc" in sys_argv
+            or "--uc-subprocess" in sys_argv
+            or "--uc_subprocess" in sys_argv
+            or "--uc-sub" in sys_argv
         )
     ):
         message = (
@@ -1363,6 +1378,9 @@ def pytest_configure(config):
     sb_config.enable_sync = config.getoption("enable_sync")
     sb_config.use_auto_ext = config.getoption("use_auto_ext")
     sb_config.undetectable = config.getoption("undetectable")
+    sb_config.uc_subprocess = config.getoption("uc_subprocess")
+    if sb_config.uc_subprocess and not sb_config.undetectable:
+        sb_config.undetectable = True
     sb_config.no_sandbox = config.getoption("no_sandbox")
     sb_config.disable_gpu = config.getoption("disable_gpu")
     sb_config.remote_debug = config.getoption("remote_debug")
@@ -1457,11 +1475,6 @@ def pytest_configure(config):
         sb_config.recorder_mode = False
         sb_config.recorder_ext = False
 
-    # Recorder Mode can still optimize scripts in --headless2 mode.
-    if sb_config.recorder_mode and sb_config.headless:
-        sb_config.headless = False
-        sb_config.headless2 = True
-
     if sb_config.xvfb and "linux" not in sys.platform:
         # The Xvfb virtual display server is for Linux OS Only!
         sb_config.xvfb = False
@@ -1480,6 +1493,12 @@ def pytest_configure(config):
             "or by calling the new --headless2.)"
         )
         sb_config.headless = True
+
+    # Recorder Mode can still optimize scripts in --headless2 mode.
+    if sb_config.recorder_mode and sb_config.headless:
+        sb_config.headless = False
+        sb_config.headless2 = True
+
     if not sb_config.headless and not sb_config.headless2:
         sb_config.headed = True
 

seleniumbase/plugins/selenium_plugin.py

@@ -68,7 +68,8 @@ class SeleniumBrowser(Plugin):
     --enable-ws  (Enable Web Security on Chromium-based browsers.)
     --enable-sync  (Enable "Chrome Sync" on websites.)
     --use-auto-ext  (Use Chrome's automation extension.)
-    --undetected | --uc  (Use undetected-chromedriver to evade bot-detection.)
+    --uc | --undetected  (Use undetected-chromedriver to evade bot-detection.)
+    --uc-sub | --uc-subprocess  (Use undetected-chromedriver as a subprocess.)
     --remote-debug  (Enable Chrome's Remote Debugger on http://localhost:9222)
     --final-debug  (Enter Debug Mode after each test ends. Don't use with CI!)
     --swiftshader  (Use Chrome's "--use-gl=swiftshader" feature.)
@@ -634,6 +635,17 @@ def options(self, parser, env):
                     to websites that use anti-bot services to block
                     automation tools from navigating them freely.""",
         )
+        parser.add_option(
+            "--uc_subprocess",
+            "--uc-subprocess",
+            "--uc-sub",  # undetected-chromedriver subprocess mode
+            action="store_true",
+            dest="uc_subprocess",
+            default=False,
+            help="""Use undetectable-chromedriver as a subprocess,
+                    which can help avoid issues that might result.
+                    It may reduce UC's ability to avoid detection.""",
+        )
         parser.add_option(
             "--no_sandbox",
             "--no-sandbox",