Merge pull request #1769 from seleniumbase/uc-mode-refactoring

Refactor UC Mode

Author: mdmintz

examples/proxy_test.py

@@ -5,6 +5,10 @@
 
 class ProxyTests(BaseCase):
     def test_proxy(self):
+        if self.headless:
+            self.open("about:blank")
+            print("Skipping test in Headless Mode.")
+            self.skip("Skipping test in Headless Mode.")
         if self.recorder_mode:
             self.open("about:blank")
             print("Skipping test in Recorder Mode.")

requirements.txt

@@ -92,7 +92,7 @@ cffi==1.15.1
 typing-extensions==4.1.1;python_version<"3.7"
 typing-extensions==4.4.0;python_version>="3.7" and python_version<"3.9"
 commonmark==0.9.1;python_version<"3.7"
-markdown-it-py==2.1.0;python_version>="3.7"
+markdown-it-py==2.2.0;python_version>="3.7"
 mdurl==0.1.2;python_version>="3.7"
 rich==12.6.0;python_version<"3.7"
 rich==13.3.1;python_version>="3.7"
@@ -101,7 +101,7 @@ rich==13.3.1;python_version>="3.7"
 # ("pip install -r requirements.txt" also installs this, but "pip install -e ." won't.)
 
 coverage==6.2;python_version<"3.7"
-coverage==7.1.0;python_version>="3.7"
+coverage==7.2.0;python_version>="3.7"
 pytest-cov==4.0.0
 flake8==5.0.4;python_version<"3.9"
 flake8==6.0.0;python_version>="3.9"

seleniumbase/__version__.py

@@ -1,2 +1,2 @@
 # seleniumbase package
-__version__ = "4.13.3"
+__version__ = "4.13.4"

seleniumbase/undetected/__init__.py

@@ -33,19 +33,8 @@
 
 
 class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
-    """
-    Controls chromedriver to drive a browser.
-    The driver gets downloaded automatically.
-
-    *** Methods ***
-    ---------------
-
-    * reconnect()
-        This can be useful when sites use heavy detection methods:
-        - Stops the chromedriver service that runs in the background.
-        - Starts the chromedriver service that runs in the background.
-        - Recreates the session.
-    """
+    """Controls chromedriver to drive a browser.
+    The driver gets downloaded automatically."""
     _instances = set()
     session_id = None
     debug = False
@@ -66,7 +55,7 @@ def __init__(
         suppress_welcome=True,
         use_subprocess=True,
         debug=False,
-        **kw
+        **kw,
     ):
         """
         Starts the Chrome service and creates a new instance of chromedriver.
@@ -216,7 +205,11 @@ def __init__(
                 language = locale.getlocale()[0].replace("_", "-")
             except Exception:
                 pass
-            if not language:
+            if (
+                not language
+                or "English" in language
+                or "United States" in language
+            ):
                 language = "en-US"
         options.add_argument("--lang=%s" % language)
         if not options.binary_location:
@@ -245,9 +238,14 @@ def __init__(
             import json
 
             with open(
-                os.path.join(user_data_dir, "Default/Preferences"),
-                encoding="latin1",
+                os.path.join(
+                    os.path.abspath(user_data_dir),
+                    "Default",
+                    "Preferences",
+                ),
+                encoding="utf-8",
                 mode="r+",
+                errors="ignore",
             ) as fs:
                 config = json.load(fs)
                 if (
@@ -405,6 +403,10 @@ def tab_list(self):
         return [PageElement(o) for o in retval]
 
     def reconnect(self, timeout=0.1):
+        """This can be useful when sites use heavy detection methods:
+        - Stops the chromedriver service that runs in the background.
+        - Starts the chromedriver service that runs in the background.
+        - Recreates the session. """
         try:
             self.service.stop()
         except Exception as e:

seleniumbase/undetected/cdp.py

@@ -74,7 +74,7 @@ def tab_close_last_opened(self):
         opentabs = [s for s in sessions if s["type"] == "page"]
         return self.post(self.endpoints["close"].format(id=opentabs[-1]["id"]))
 
-    async def send(self, method, params):  # noqa
+    async def send(self, method, params):
         import websockets
 
         self._reqid += 1

seleniumbase/undetected/dprocess.py

@@ -11,20 +11,17 @@
 
 
 def start_detached(executable, *args):
-    """
-    Starts a fully independent subprocess with no parent.
+    """Starts a fully independent subprocess with no parent.
     :param executable: executable
     :param args: arguments to the executable,
         eg: ["--param1_key=param1_val", "-vvv"]
-    :return: pid of the grandchild process
-    """
+    :return: pid of the grandchild process """
     import multiprocessing
 
     reader, writer = multiprocessing.Pipe(False)  # Create pipe
-    # Do not keep reference
     multiprocessing.Process(
         target=_start_detached,
-        args=(executable, *args),  # noqa
+        args=(executable, *args),
         kwargs={"writer": writer},
         daemon=True,
     ).start()

seleniumbase/undetected/options.py

@@ -51,7 +51,9 @@ def handle_prefs(self, user_data_dir):
             prefs_file = os.path.join(default_path, "Preferences")
             try:
                 if os.path.exists(prefs_file):
-                    with open(prefs_file, encoding="utf-8", mode="r") as f:
+                    with open(
+                        prefs_file, encoding="utf-8", mode="r", errors="ignore"
+                    ) as f:
                         undot_prefs = self._merge_nested(
                             json.load(f), undot_prefs
                         )

seleniumbase/undetected/patcher.py

@@ -39,15 +39,13 @@ class Patcher(object):
     data_path = os.path.abspath(os.path.expanduser(downloads_folder))
 
     def __init__(self, executable_path=None, force=False, version_main=0):
-        """
-        Args:
-            executable_path: None = automatic
-                A full file path to the chromedriver executable
-            force: False
-                Terminate processes which are holding lock
-            version_main: 0 = auto
-                Specify main chrome version (rounded, ex: 82)
-        """
+        """Args:
+        executable_path: None = automatic
+            A full file path to the chromedriver executable
+        force: False
+            Terminate processes which are holding lock
+        version_main: 0 = auto
+            Specify main chrome version (rounded, ex: 82) """
         self.force = force
         self.executable_path = None
         prefix = "undetected"
@@ -89,17 +87,17 @@ def auto(self, executable_path=None, force=False, version_main=None):
             self.force = force
         try:
             os.unlink(self.executable_path)
-        except PermissionError:  # noqa
+        except PermissionError:
             if self.force:
                 self.force_kill_instances(self.executable_path)
                 not_force = not self.force
                 return self.auto(force=not_force)
             try:
                 if self.is_binary_patched():
                     return True  # Running AND patched
-            except PermissionError:  # noqa
+            except PermissionError:
                 pass
-        except FileNotFoundError:  # noqa
+        except FileNotFoundError:
             pass
         release = self.fetch_release_number()
         self.version_main = release.split(".")[0]
@@ -122,10 +120,8 @@ def fetch_release_number(self):
         return urlopen(self.url_repo + path).read().decode()
 
     def fetch_package(self):
-        """
-        Downloads chromedriver from source.
-        :return: path to downloaded file
-        """
+        """Downloads chromedriver from source.
+        :return: path to downloaded file """
         from urllib.request import urlretrieve
 
         u = "%s/%s/%s" % (
@@ -135,13 +131,11 @@ def fetch_package(self):
         return urlretrieve(u)[0]
 
     def unzip_package(self, fp):
-        """
-        :return: path to unpacked executable
-        """
+        """ :return: path to unpacked executable """
         logger.debug("unzipping %s" % fp)
         try:
             os.unlink(self.zip_path)
-        except (FileNotFoundError, OSError):  # noqa
+        except (FileNotFoundError, OSError):
             pass
         os.makedirs(self.zip_path, mode=0o755, exist_ok=True)
         with zipfile.ZipFile(fp, mode="r") as zf:
@@ -163,10 +157,9 @@ def unzip_package(self, fp):
 
     @staticmethod
     def force_kill_instances(exe_name):
-        """
+        """ Terminate instances of UC.
         :param: executable name to kill, may be a path as well
-        :return: True on success else False
-        """
+        :return: True on success else False """
         exe_name = os.path.basename(exe_name)
         if IS_POSIX:
             r = os.system("kill -f -9 $(pidof %s)" % exe_name)
@@ -288,8 +281,8 @@ def __del__(self):
                         % self.executable_path
                     )
                     break
-                except (OSError, RuntimeError, PermissionError):  # noqa
+                except (OSError, RuntimeError, PermissionError):
                     time.sleep(0.1)
                     continue
-                except FileNotFoundError:  # noqa
+                except FileNotFoundError:
                     break

seleniumbase/undetected/reactor.py

@@ -41,7 +41,7 @@ def run(self):
         except Exception as e:
             logger.warning("Reactor.run() => %s", e)
 
-    async def _wait_service_started(self):  # noqa
+    async def _wait_service_started(self):
         while True:
             with self.lock:
                 if (
@@ -53,7 +53,7 @@ async def _wait_service_started(self):  # noqa
                 else:
                     break
 
-    async def listen(self):  # noqa
+    async def listen(self):
         while self.running:
             await self._wait_service_started()
             await asyncio.sleep(1)

setup.py

@@ -216,7 +216,7 @@
         'typing-extensions==4.1.1;python_version<"3.7"',
         'typing-extensions==4.4.0;python_version>="3.7" and python_version<"3.9"',  # noqa: E501
         'commonmark==0.9.1;python_version<"3.7"',  # For old "rich"
-        'markdown-it-py==2.1.0;python_version>="3.7"',  # For new "rich"
+        'markdown-it-py==2.2.0;python_version>="3.7"',  # For new "rich"
         'mdurl==0.1.2;python_version>="3.7"',  # For new "rich"
         'rich==12.6.0;python_version<"3.7"',
         'rich==13.3.1;python_version>="3.7"',
@@ -226,7 +226,7 @@
         # Usage: coverage run -m pytest; coverage html; coverage report
         "coverage": [
             'coverage==6.2;python_version<"3.7"',
-            'coverage==7.1.0;python_version>="3.7"',
+            'coverage==7.2.0;python_version>="3.7"',
             "pytest-cov==4.0.0",
         ],
         # pip install -e .[flake8]