Merge pull request #1382 from seleniumbase/proxy-pac-features-and-bug-fixes

Proxy PAC features and bug fixes

Author: mdmintz

README.md

@@ -440,6 +440,8 @@ The code above will leave your browser window open in case there's a failure. (i
 --proxy=SERVER:PORT  # (Connect to a proxy server:port for tests.)
 --proxy=USERNAME:PASSWORD@SERVER:PORT  # (Use authenticated proxy server.)
 --proxy-bypass-list=STRING  # (";"-separated hosts to bypass, Eg "*.foo.com")
+--proxy-pac-url=URL  # (Connect to a proxy server using a PAC_URL.pac file.)
+--proxy-pac-url=USERNAME:PASSWORD@URL  # (Authenticated proxy with PAC URL.)
 --agent=STRING  # (Modify the web browser's User-Agent string.)
 --mobile  # (Use the mobile device emulator while running tests.)
 --metrics=STRING  # (Set mobile metrics: "CSSWidth,CSSHeight,PixelRatio".)
@@ -963,13 +965,13 @@ def get_mirror_universe_captain_picard_superbowl_ad(superbowl_year):
 self.switch_to_window(1)  # This switches to the new tab (0 is the first one)
 ```
 
-🔵 <b>ProTip™:</b> iFrames follow the same principle as new windows - you need to specify the iFrame if you want to take action on something in there
+🔵 <b>ProTip™:</b> iframes follow the same principle as new windows - you need to specify the iframe if you want to take action on something in there
 
 ```python
 self.switch_to_frame('ContentManagerTextBody_ifr')
-# Now you can act inside the iFrame
+# Now you can act inside the iframe
 # .... Do something cool (here)
-self.switch_to_default_content()  # Exit the iFrame when you're done
+self.switch_to_default_content()  # Exit the iframe when you're done
 ```
 
 🔵 Executing Custom jQuery Scripts:
@@ -1109,3 +1111,4 @@ pytest --reruns=1 --reruns-delay=1
 
 <p><a href="https://github.com/seleniumbase/SeleniumBase/"><img src="https://seleniumbase.io/cdn/img/super_logo_sb.png" alt="SeleniumBase" title="SeleniumBase" width="200" /></a></p>
 <p><a href="https://pypi.python.org/pypi/seleniumbase" target="_blank"><img src="https://badge.fury.io/py/seleniumbase.svg" alt="PyPI version" /></a></p>
+<p><a href="https://www.python.org/downloads/" target="_blank"><img src="https://img.shields.io/pypi/pyversions/seleniumbase.svg?color=22AAEE&logo=python" title="Supported Python Versions" /></a></p>

examples/raw_parameter_script.py

@@ -102,6 +102,7 @@
     sb.firefox_pref = None
     sb.proxy_string = None
     sb.proxy_bypass_list = None
+    sb.proxy_pac_url = None
     sb.swiftshader = False
     sb.ad_block_on = False
     sb.highlights = None

help_docs/customizing_test_runs.md

@@ -122,6 +122,8 @@ pytest my_first_test.py --settings-file=custom_settings.py
 --proxy=SERVER:PORT  # (Connect to a proxy server:port for tests.)
 --proxy=USERNAME:PASSWORD@SERVER:PORT  # (Use authenticated proxy server.)
 --proxy-bypass-list=STRING  # (";"-separated hosts to bypass, Eg "*.foo.com")
+--proxy-pac-url=URL  # (Connect to a proxy server using a PAC_URL.pac file.)
+--proxy-pac-url=USERNAME:PASSWORD@URL  # (Authenticated proxy with PAC URL.)
 --agent=STRING  # (Modify the web browser's User-Agent string.)
 --mobile  # (Use the mobile device emulator while running tests.)
 --metrics=STRING  # (Set mobile metrics: "CSSWidth,CSSHeight,PixelRatio".)

help_docs/features_list.md

@@ -9,18 +9,20 @@
 * Supports multiple browsers, tabs, iframes, and proxies in the same test.
 * Automatic smart-waiting improves reliability and prevents flaky tests.
 * Supports [pytest](https://docs.pytest.org/en/latest/), [unittest](https://docs.python.org/3/library/unittest.html), [nose](http://nose.readthedocs.io/en/latest/), and [behave](https://behave.readthedocs.io/en/stable/index.html) for finding/running tests.
-* All the code is open source. Dive in to see details about any feature.
+* All the code is open source. Look inside to learn about any feature.
 * Powerful logging tools for [dashboards, reports, and screenshots](https://github.com/seleniumbase/SeleniumBase/blob/master/examples/example_logs/ReadMe.md).
 * Can run tests in Headless Mode to hide the browser. (``--headless``)
 * Can run tests multithreaded from parallel browsers. (``-n NUM_THREADS``)
 * Can run tests from a shared browser session. (``--reuse-session``/``--rs``)
 * Can run tests using Chromium's mobile device emulator. (``--mobile``)
 * Can run tests through a proxy server. (``--proxy=IP_ADDRESS:PORT``)
+* Can run tests with proxy settings via PAC URL. (``--proxy-pac-url=URL.pac``)
 * Can run tests through an authenticated proxy server. (``--proxy=USER:PASS@HOST:PORT``)
+* Can run tests with proxy+auth via PAC URL. (``--proxy-pac-url=USER:[email protected]``)
 * Can run tests with a customized browser user agent. (``--agent=USER_AGENT_STRING``)
-* Can set a Chromium User Data Directory/Profile to load. (``--user_data_dir=DIR``)
-* Can load Chrome Extension ZIP files. (``--extension_zip=ZIP``)
-* Can load Chrome Extension folders. (``--extension_dir=DIR``)
+* Can set a Chromium User Data Directory/Profile to load. (``--user-data-dir=DIR``)
+* Can load Chrome Extension ZIP files. (``--extension-zip=ZIP``)
+* Can load Chrome Extension folders. (``--extension-dir=DIR``)
 * Powerful [console scripts](https://github.com/seleniumbase/SeleniumBase/blob/master/seleniumbase/console_scripts/ReadMe.md). (Type **``seleniumbase``** or **``sbase``** to use.)
 * Has the ability to translate tests into [multiple spoken languages](https://github.com/seleniumbase/SeleniumBase/tree/master/examples/translations).
 * Has a flexible [command-line interface](https://github.com/seleniumbase/SeleniumBase/blob/master/help_docs/customizing_test_runs.md) for customizing test runs.

mkdocs_build/requirements.txt

@@ -19,7 +19,7 @@ ghp-import==2.1.0
 readme-renderer==35.0
 pymdown-extensions==9.5
 importlib-metadata==4.12.0
-bleach==5.0.0
+bleach==5.0.1
 jsmin==3.0.1
 lunr==0.6.2
 nltk==3.7

requirements.txt

@@ -14,8 +14,7 @@ attrs>=21.4.0
 PyYAML>=6.0;python_version>="3.6"
 traitlets>=4.3.3;python_version<"3.7"
 traitlets>=5.3.0;python_version>="3.7"
-certifi>=2021.10.8;python_version<"3.5"
-certifi>=2022.5.18;python_version>="3.5" and python_version<"3.6"
+certifi>=2021.10.8;python_version<"3.6"
 certifi>=2022.6.15;python_version>="3.6"
 filelock>=3.2.1;python_version<"3.6"
 filelock>=3.4.1;python_version>="3.6" and python_version<"3.7"
@@ -101,7 +100,7 @@ pygments==2.11.2;python_version>="3.5" and python_version<"3.6"
 pygments==2.12.0;python_version>="3.6"
 prompt-toolkit==1.0.18;python_version<"3.5"
 prompt-toolkit==2.0.10;python_version>="3.5" and python_version<"3.6"
-prompt-toolkit==3.0.29;python_version>="3.6"
+prompt-toolkit==3.0.30;python_version>="3.6"
 decorator==4.4.2;python_version<"3.5"
 decorator==5.1.1;python_version>="3.5"
 ipython==5.10.0;python_version<"3.5"

seleniumbase/__version__.py

@@ -1,2 +1,2 @@
 # seleniumbase package
-__version__ = "3.3.2"
+__version__ = "3.3.3"

seleniumbase/behave/behave_helper.py

@@ -17,7 +17,14 @@ def generate_gherkin(srt_actions):
                     action[2] = unquote(action[2], errors="strict")
                 except Exception:
                     pass
-            sb_actions.append('Open "%s"' % action[2])
+            if '"' not in action[2]:
+                sb_actions.append('Open "%s"' % action[2])
+            elif "'" not in action[2]:
+                sb_actions.append("Open '%s'" % action[2])
+            else:
+                sb_actions.append(
+                    'Open "%s"' % action[2].replace('"', '\\"')
+                )
         elif action[0] == "f_url":
             if "%" in action[2] and python3:
                 try:
@@ -26,7 +33,14 @@ def generate_gherkin(srt_actions):
                     action[2] = unquote(action[2], errors="strict")
                 except Exception:
                     pass
-            sb_actions.append('Open if not "%s"' % action[2])
+            if '"' not in action[2]:
+                sb_actions.append('Open if not "%s"' % action[2])
+            elif "'" not in action[2]:
+                sb_actions.append("Open if not '%s'" % action[2])
+            else:
+                sb_actions.append(
+                    'Open if not "%s"' % action[2].replace('"', '\\"')
+                )
         elif action[0] == "click":
             if '"' not in action[1]:
                 sb_actions.append('Click "%s"' % action[1])

seleniumbase/behave/behave_sb.py

@@ -31,6 +31,8 @@
 -D proxy=SERVER:PORT  (Connect to a proxy server:port for tests.)
 -D proxy=USERNAME:PASSWORD@SERVER:PORT  (Use authenticated proxy server.)
 -D proxy-bypass-list=STRING (";"-separated hosts to bypass, Eg "*.foo.com")
+-D proxy-pac-url=URL  (Connect to a proxy server using a PAC_URL.pac file.)
+-D proxy-pac-url=USERNAME:PASSWORD@URL  (Authenticated proxy with PAC URL.)
 -D agent=STRING  (Modify the web browser's User-Agent string.)
 -D mobile  (Use the mobile device emulator while running tests.)
 -D metrics=STRING  (Set mobile metrics: "CSSWidth,CSSHeight,PixelRatio".)
@@ -195,6 +197,7 @@ def get_configured_sb(context):
     sb.firefox_pref = None
     sb.proxy_string = None
     sb.proxy_bypass_list = None
+    sb.proxy_pac_url = None
     sb.swiftshader = False
     sb.ad_block_on = False
     sb.highlights = None
@@ -619,7 +622,7 @@ def get_configured_sb(context):
             sb.firefox_pref = firefox_pref
             continue
         # Handle: -D proxy=SERVER:PORT / proxy=USERNAME:PASSWORD@SERVER:PORT
-        if low_key == "proxy":
+        if low_key in ["proxy", "proxy-server", "proxy-string"]:
             proxy_string = userdata[key]
             if proxy_string == "true":
                 proxy_string = sb.proxy_string  # revert to default
@@ -632,6 +635,13 @@ def get_configured_sb(context):
                 proxy_bypass_list = sb.proxy_bypass_list  # revert to default
             sb.proxy_bypass_list = proxy_bypass_list
             continue
+        # Handle: -D proxy-pac-url=URL / proxy-pac-url=USERNAME:PASSWORD@URL
+        if low_key in ["proxy-pac-url", "proxy_pac_url", "pac-url", "pac_url"]:
+            proxy_pac_url = userdata[key]
+            if proxy_pac_url == "true":
+                proxy_pac_url = sb.proxy_pac_url  # revert to default
+            sb.proxy_pac_url = proxy_pac_url
+            continue
         # Handle: -D swiftshader
         if low_key == "swiftshader":
             sb.swiftshader = True

seleniumbase/core/browser_launcher.py

@@ -261,6 +261,7 @@ def _set_chrome_options(
     proxy_user,
     proxy_pass,
     proxy_bypass_list,
+    proxy_pac_url,
     user_agent,
     recorder_ext,
     disable_csp,
@@ -458,6 +459,12 @@ def _set_chrome_options(
             chrome_options.add_argument(
                 "--proxy-bypass-list=%s" % proxy_bypass_list
             )
+    elif proxy_pac_url:
+        if proxy_auth:
+            chrome_options = _add_chrome_proxy_extension(
+                chrome_options, None, proxy_user, proxy_pass
+            )
+        chrome_options.add_argument("--proxy-pac-url=%s" % proxy_pac_url)
     if headless:
         if not proxy_auth and not browser_name == constants.Browser.OPERA:
             # Headless Chrome doesn't support extensions, which are
@@ -509,6 +516,7 @@ def _set_firefox_options(
     locale_code,
     proxy_string,
     proxy_bypass_list,
+    proxy_pac_url,
     user_agent,
     disable_csp,
     firefox_arg,
@@ -565,6 +573,9 @@ def _set_firefox_options(
             options.set_preference("network.proxy.ssl_port", int(proxy_port))
         if proxy_bypass_list:
             options.set_preference("no_proxies_on", proxy_bypass_list)
+    elif proxy_pac_url:
+        options.set_preference("network.proxy.type", 2)
+        options.set_preference("network.proxy.autoconfig_url", proxy_pac_url)
     if user_agent:
         options.set_preference("general.useragent.override", user_agent)
     options.set_preference(
@@ -716,6 +727,7 @@ def get_driver(
     port=4444,
     proxy_string=None,
     proxy_bypass_list=None,
+    proxy_pac_url=None,
     user_agent=None,
     cap_file=None,
     cap_string=None,
@@ -775,6 +787,33 @@ def get_driver(
         proxy_string = validate_proxy_string(proxy_string)
         if proxy_string and proxy_user and proxy_pass:
             proxy_auth = True
+    elif proxy_pac_url:
+        username_and_password = None
+        if "@" in proxy_pac_url:
+            # Format => username:password@PAC_URL.pac
+            try:
+                username_and_password = proxy_pac_url.split("@")[0]
+                proxy_pac_url = proxy_pac_url.split("@")[1]
+                proxy_user = username_and_password.split(":")[0]
+                proxy_pass = username_and_password.split(":")[1]
+            except Exception:
+                raise Exception(
+                    "The format for using a PAC URL with authentication "
+                    'is: "username:password@PAC_URL.pac". If using a PAC '
+                    'URL without auth, the format is: "PAC_URL.pac".'
+                )
+            if browser_name != constants.Browser.GOOGLE_CHROME and (
+                browser_name != constants.Browser.EDGE
+            ):
+                raise Exception(
+                    "Chrome or Edge is required when using a PAC URL "
+                    "that has authentication! (If using a PAC URL "
+                    "without auth, Chrome, Edge, or Firefox may be used.)"
+                )
+        if not proxy_pac_url.lower().endswith(".pac"):
+            raise Exception('The proxy PAC URL must end with ".pac"!')
+        if proxy_pac_url and proxy_user and proxy_pass:
+            proxy_auth = True
     if browser_name == "chrome" and user_data_dir and len(user_data_dir) < 3:
         raise Exception(
             "Name length of Chrome's User Data Directory must be >= 3."
@@ -792,6 +831,7 @@ def get_driver(
             proxy_user,
             proxy_pass,
             proxy_bypass_list,
+            proxy_pac_url,
             user_agent,
             cap_file,
             cap_string,
@@ -833,6 +873,7 @@ def get_driver(
             proxy_user,
             proxy_pass,
             proxy_bypass_list,
+            proxy_pac_url,
             user_agent,
             recorder_ext,
             disable_csp,
@@ -874,6 +915,7 @@ def get_remote_driver(
     proxy_user,
     proxy_pass,
     proxy_bypass_list,
+    proxy_pac_url,
     user_agent,
     cap_file,
     cap_string,
@@ -969,6 +1011,7 @@ def get_remote_driver(
             proxy_user,
             proxy_pass,
             proxy_bypass_list,
+            proxy_pac_url,
             user_agent,
             recorder_ext,
             disable_csp,
@@ -1050,6 +1093,7 @@ def get_remote_driver(
             locale_code,
             proxy_string,
             proxy_bypass_list,
+            proxy_pac_url,
             user_agent,
             disable_csp,
             firefox_arg,
@@ -1175,6 +1219,7 @@ def get_remote_driver(
             proxy_user,
             proxy_pass,
             proxy_bypass_list,
+            proxy_pac_url,
             user_agent,
             recorder_ext,
             disable_csp,
@@ -1356,6 +1401,7 @@ def get_local_driver(
     proxy_user,
     proxy_pass,
     proxy_bypass_list,
+    proxy_pac_url,
     user_agent,
     recorder_ext,
     disable_csp,
@@ -1396,6 +1442,7 @@ def get_local_driver(
             locale_code,
             proxy_string,
             proxy_bypass_list,
+            proxy_pac_url,
             user_agent,
             disable_csp,
             firefox_arg,
@@ -1704,10 +1751,16 @@ def get_local_driver(
                     edge_options, proxy_string, proxy_user, proxy_pass
                 )
             edge_options.add_argument("--proxy-server=%s" % proxy_string)
-        if proxy_bypass_list:
-            edge_options.add_argument(
-                "--proxy-bypass-list=%s" % proxy_bypass_list
-            )
+            if proxy_bypass_list:
+                edge_options.add_argument(
+                    "--proxy-bypass-list=%s" % proxy_bypass_list
+                )
+        elif proxy_pac_url:
+            if proxy_auth:
+                edge_options = _add_chrome_proxy_extension(
+                    edge_options, None, proxy_user, proxy_pass
+                )
+            edge_options.add_argument("--proxy-pac-url=%s" % proxy_pac_url)
         edge_options.add_argument("--test-type")
         edge_options.add_argument("--log-level=3")
         edge_options.add_argument("--no-first-run")
@@ -1862,6 +1915,7 @@ def get_local_driver(
                 proxy_user,
                 proxy_pass,
                 proxy_bypass_list,
+                proxy_pac_url,
                 user_agent,
                 recorder_ext,
                 disable_csp,
@@ -1917,6 +1971,7 @@ def get_local_driver(
                 proxy_user,
                 proxy_pass,
                 proxy_bypass_list,
+                proxy_pac_url,
                 user_agent,
                 recorder_ext,
                 disable_csp,
@@ -2036,6 +2091,7 @@ def get_local_driver(
                         proxy_user,
                         proxy_pass,
                         proxy_bypass_list,
+                        proxy_pac_url,
                         user_agent,
                         recorder_ext,
                         disable_csp,