Add option to set proxy settings via PAC URL

Author: mdmintz

examples/raw_parameter_script.py

@@ -102,6 +102,7 @@
     sb.firefox_pref = None
     sb.proxy_string = None
     sb.proxy_bypass_list = None
+    sb.proxy_pac_url = None
     sb.swiftshader = False
     sb.ad_block_on = False
     sb.highlights = None

seleniumbase/behave/behave_sb.py

@@ -31,6 +31,8 @@
 -D proxy=SERVER:PORT  (Connect to a proxy server:port for tests.)
 -D proxy=USERNAME:PASSWORD@SERVER:PORT  (Use authenticated proxy server.)
 -D proxy-bypass-list=STRING (";"-separated hosts to bypass, Eg "*.foo.com")
+-D proxy-pac-url=URL  (Connect to a proxy server using a PAC_URL.pac file.)
+-D proxy-pac-url=USERNAME:PASSWORD@URL  (Authenticated proxy with PAC URL.)
 -D agent=STRING  (Modify the web browser's User-Agent string.)
 -D mobile  (Use the mobile device emulator while running tests.)
 -D metrics=STRING  (Set mobile metrics: "CSSWidth,CSSHeight,PixelRatio".)
@@ -195,6 +197,7 @@ def get_configured_sb(context):
     sb.firefox_pref = None
     sb.proxy_string = None
     sb.proxy_bypass_list = None
+    sb.proxy_pac_url = None
     sb.swiftshader = False
     sb.ad_block_on = False
     sb.highlights = None
@@ -619,7 +622,7 @@ def get_configured_sb(context):
             sb.firefox_pref = firefox_pref
             continue
         # Handle: -D proxy=SERVER:PORT / proxy=USERNAME:PASSWORD@SERVER:PORT
-        if low_key == "proxy":
+        if low_key in ["proxy", "proxy-server", "proxy-string"]:
             proxy_string = userdata[key]
             if proxy_string == "true":
                 proxy_string = sb.proxy_string  # revert to default
@@ -632,6 +635,13 @@ def get_configured_sb(context):
                 proxy_bypass_list = sb.proxy_bypass_list  # revert to default
             sb.proxy_bypass_list = proxy_bypass_list
             continue
+        # Handle: -D proxy-pac-url=URL / proxy-pac-url=USERNAME:PASSWORD@URL
+        if low_key in ["proxy-pac-url", "proxy_pac_url", "pac-url", "pac_url"]:
+            proxy_pac_url = userdata[key]
+            if proxy_pac_url == "true":
+                proxy_pac_url = sb.proxy_pac_url  # revert to default
+            sb.proxy_pac_url = proxy_pac_url
+            continue
         # Handle: -D swiftshader
         if low_key == "swiftshader":
             sb.swiftshader = True

seleniumbase/core/browser_launcher.py

@@ -261,6 +261,7 @@ def _set_chrome_options(
     proxy_user,
     proxy_pass,
     proxy_bypass_list,
+    proxy_pac_url,
     user_agent,
     recorder_ext,
     disable_csp,
@@ -458,6 +459,12 @@ def _set_chrome_options(
             chrome_options.add_argument(
                 "--proxy-bypass-list=%s" % proxy_bypass_list
             )
+    elif proxy_pac_url:
+        if proxy_auth:
+            chrome_options = _add_chrome_proxy_extension(
+                chrome_options, None, proxy_user, proxy_pass
+            )
+        chrome_options.add_argument("--proxy-pac-url=%s" % proxy_pac_url)
     if headless:
         if not proxy_auth and not browser_name == constants.Browser.OPERA:
             # Headless Chrome doesn't support extensions, which are
@@ -509,6 +516,7 @@ def _set_firefox_options(
     locale_code,
     proxy_string,
     proxy_bypass_list,
+    proxy_pac_url,
     user_agent,
     disable_csp,
     firefox_arg,
@@ -565,6 +573,9 @@ def _set_firefox_options(
             options.set_preference("network.proxy.ssl_port", int(proxy_port))
         if proxy_bypass_list:
             options.set_preference("no_proxies_on", proxy_bypass_list)
+    elif proxy_pac_url:
+        options.set_preference("network.proxy.type", 2)
+        options.set_preference("network.proxy.autoconfig_url", proxy_pac_url)
     if user_agent:
         options.set_preference("general.useragent.override", user_agent)
     options.set_preference(
@@ -716,6 +727,7 @@ def get_driver(
     port=4444,
     proxy_string=None,
     proxy_bypass_list=None,
+    proxy_pac_url=None,
     user_agent=None,
     cap_file=None,
     cap_string=None,
@@ -775,6 +787,33 @@ def get_driver(
         proxy_string = validate_proxy_string(proxy_string)
         if proxy_string and proxy_user and proxy_pass:
             proxy_auth = True
+    elif proxy_pac_url:
+        username_and_password = None
+        if "@" in proxy_pac_url:
+            # Format => username:password@PAC_URL.pac
+            try:
+                username_and_password = proxy_pac_url.split("@")[0]
+                proxy_pac_url = proxy_pac_url.split("@")[1]
+                proxy_user = username_and_password.split(":")[0]
+                proxy_pass = username_and_password.split(":")[1]
+            except Exception:
+                raise Exception(
+                    "The format for using a PAC URL with authentication "
+                    'is: "username:password@PAC_URL.pac". If using a PAC '
+                    'URL without auth, the format is: "PAC_URL.pac".'
+                )
+            if browser_name != constants.Browser.GOOGLE_CHROME and (
+                browser_name != constants.Browser.EDGE
+            ):
+                raise Exception(
+                    "Chrome or Edge is required when using a PAC URL "
+                    "that has authentication! (If using a PAC URL "
+                    "without auth, Chrome, Edge, or Firefox may be used.)"
+                )
+        if not proxy_pac_url.lower().endswith(".pac"):
+            raise Exception('The proxy PAC URL must end with ".pac"!')
+        if proxy_pac_url and proxy_user and proxy_pass:
+            proxy_auth = True
     if browser_name == "chrome" and user_data_dir and len(user_data_dir) < 3:
         raise Exception(
             "Name length of Chrome's User Data Directory must be >= 3."
@@ -792,6 +831,7 @@ def get_driver(
             proxy_user,
             proxy_pass,
             proxy_bypass_list,
+            proxy_pac_url,
             user_agent,
             cap_file,
             cap_string,
@@ -833,6 +873,7 @@ def get_driver(
             proxy_user,
             proxy_pass,
             proxy_bypass_list,
+            proxy_pac_url,
             user_agent,
             recorder_ext,
             disable_csp,
@@ -874,6 +915,7 @@ def get_remote_driver(
     proxy_user,
     proxy_pass,
     proxy_bypass_list,
+    proxy_pac_url,
     user_agent,
     cap_file,
     cap_string,
@@ -969,6 +1011,7 @@ def get_remote_driver(
             proxy_user,
             proxy_pass,
             proxy_bypass_list,
+            proxy_pac_url,
             user_agent,
             recorder_ext,
             disable_csp,
@@ -1050,6 +1093,7 @@ def get_remote_driver(
             locale_code,
             proxy_string,
             proxy_bypass_list,
+            proxy_pac_url,
             user_agent,
             disable_csp,
             firefox_arg,
@@ -1175,6 +1219,7 @@ def get_remote_driver(
             proxy_user,
             proxy_pass,
             proxy_bypass_list,
+            proxy_pac_url,
             user_agent,
             recorder_ext,
             disable_csp,
@@ -1356,6 +1401,7 @@ def get_local_driver(
     proxy_user,
     proxy_pass,
     proxy_bypass_list,
+    proxy_pac_url,
     user_agent,
     recorder_ext,
     disable_csp,
@@ -1396,6 +1442,7 @@ def get_local_driver(
             locale_code,
             proxy_string,
             proxy_bypass_list,
+            proxy_pac_url,
             user_agent,
             disable_csp,
             firefox_arg,
@@ -1704,10 +1751,16 @@ def get_local_driver(
                     edge_options, proxy_string, proxy_user, proxy_pass
                 )
             edge_options.add_argument("--proxy-server=%s" % proxy_string)
-        if proxy_bypass_list:
-            edge_options.add_argument(
-                "--proxy-bypass-list=%s" % proxy_bypass_list
-            )
+            if proxy_bypass_list:
+                edge_options.add_argument(
+                    "--proxy-bypass-list=%s" % proxy_bypass_list
+                )
+        elif proxy_pac_url:
+            if proxy_auth:
+                edge_options = _add_chrome_proxy_extension(
+                    edge_options, None, proxy_user, proxy_pass
+                )
+            edge_options.add_argument("--proxy-pac-url=%s" % proxy_pac_url)
         edge_options.add_argument("--test-type")
         edge_options.add_argument("--log-level=3")
         edge_options.add_argument("--no-first-run")
@@ -1862,6 +1915,7 @@ def get_local_driver(
                 proxy_user,
                 proxy_pass,
                 proxy_bypass_list,
+                proxy_pac_url,
                 user_agent,
                 recorder_ext,
                 disable_csp,
@@ -1917,6 +1971,7 @@ def get_local_driver(
                 proxy_user,
                 proxy_pass,
                 proxy_bypass_list,
+                proxy_pac_url,
                 user_agent,
                 recorder_ext,
                 disable_csp,
@@ -2036,6 +2091,7 @@ def get_local_driver(
                         proxy_user,
                         proxy_pass,
                         proxy_bypass_list,
+                        proxy_pac_url,
                         user_agent,
                         recorder_ext,
                         disable_csp,

seleniumbase/core/proxy_helper.py

@@ -11,39 +11,65 @@ def create_proxy_zip(proxy_string, proxy_user, proxy_pass):
     """Implementation of https://stackoverflow.com/a/35293284 for
     https://stackoverflow.com/questions/12848327/
     (Run Selenium on a proxy server that requires authentication.)
-    Solution involves creating & adding a Chrome extension on the fly.
-    * CHROME-ONLY for now! *
+    Solution involves creating & adding a Chromium extension on the fly.
+    CHROMIUM-ONLY! *** Only Chrome and Edge browsers are supported. ***
     """
-    proxy_host = proxy_string.split(":")[0]
-    proxy_port = proxy_string.split(":")[1]
-    background_js = (
-        """var config = {\n"""
-        """    mode: "fixed_servers",\n"""
-        """    rules: {\n"""
-        """      singleProxy: {\n"""
-        """        scheme: "http",\n"""
-        """        host: "%s",\n"""
-        """        port: parseInt("%s")\n"""
-        """      },\n"""
-        """    }\n"""
-        """  };\n"""
-        """chrome.proxy.settings.set("""
-        """{value: config, scope: "regular"}, function() {"""
-        """});\n"""
-        """function callbackFn(details) {\n"""
-        """    return {\n"""
-        """        authCredentials: {\n"""
-        """            username: "%s",\n"""
-        """            password: "%s"\n"""
-        """        }\n"""
-        """    };\n"""
-        """}\n"""
-        """chrome.webRequest.onAuthRequired.addListener(\n"""
-        """        callbackFn,\n"""
-        """        {urls: ["<all_urls>"]},\n"""
-        """        ['blocking']\n"""
-        """);""" % (proxy_host, proxy_port, proxy_user, proxy_pass)
-    )
+    background_js = None
+    if proxy_string:
+        proxy_host = proxy_string.split(":")[0]
+        proxy_port = proxy_string.split(":")[1]
+        background_js = (
+            """var config = {\n"""
+            """    mode: "fixed_servers",\n"""
+            """    rules: {\n"""
+            """      singleProxy: {\n"""
+            """        scheme: "http",\n"""
+            """        host: "%s",\n"""
+            """        port: parseInt("%s")\n"""
+            """      },\n"""
+            """    }\n"""
+            """  };\n"""
+            """chrome.proxy.settings.set("""
+            """{value: config, scope: "regular"}, function() {"""
+            """});\n"""
+            """function callbackFn(details) {\n"""
+            """    return {\n"""
+            """        authCredentials: {\n"""
+            """            username: "%s",\n"""
+            """            password: "%s"\n"""
+            """        }\n"""
+            """    };\n"""
+            """}\n"""
+            """chrome.webRequest.onAuthRequired.addListener(\n"""
+            """        callbackFn,\n"""
+            """        {urls: ["<all_urls>"]},\n"""
+            """        ['blocking']\n"""
+            """);""" % (proxy_host, proxy_port, proxy_user, proxy_pass)
+        )
+    else:
+        background_js = (
+            """var config = {\n"""
+            """    mode: "fixed_servers",\n"""
+            """    rules: {\n"""
+            """    }\n"""
+            """  };\n"""
+            """chrome.proxy.settings.set("""
+            """{value: config, scope: "regular"}, function() {"""
+            """});\n"""
+            """function callbackFn(details) {\n"""
+            """    return {\n"""
+            """        authCredentials: {\n"""
+            """            username: "%s",\n"""
+            """            password: "%s"\n"""
+            """        }\n"""
+            """    };\n"""
+            """}\n"""
+            """chrome.webRequest.onAuthRequired.addListener(\n"""
+            """        callbackFn,\n"""
+            """        {urls: ["<all_urls>"]},\n"""
+            """        ['blocking']\n"""
+            """);""" % (proxy_user, proxy_pass)
+        )
     manifest_json = (
         """{\n"""
         """"version": "1.0.0",\n"""
@@ -79,7 +105,7 @@ def create_proxy_zip(proxy_string, proxy_user, proxy_pass):
 
 
 def remove_proxy_zip_if_present():
-    """Remove Chrome extension zip file used for proxy server authentication.
+    """Remove Chromium extension zip file used for proxy server authentication.
     Used in the implementation of https://stackoverflow.com/a/35293284
     for https://stackoverflow.com/questions/12848327/
     """

seleniumbase/fixtures/base_case.py

@@ -2937,6 +2937,7 @@ def get_new_driver(
         port=None,
         proxy=None,
         proxy_bypass_list=None,
+        proxy_pac_url=None,
         agent=None,
         switch_to=True,
         cap_file=None,
@@ -2980,6 +2981,7 @@ def get_new_driver(
         port - if using a Selenium Grid, set the host port here
         proxy - if using a proxy server, specify the "host:port" combo here
         proxy_bypass_list - ";"-separated hosts to bypass (Eg. "*.foo.com")
+        proxy_pac_url - designates the proxy PAC URL to use (Chromium-only)
         switch_to - the option to switch to the new driver (default = True)
         cap_file - the file containing desired capabilities for the browser
         cap_string - the string with desired capabilities for the browser
@@ -3060,6 +3062,8 @@ def get_new_driver(
             proxy_string = self.proxy_string
         if proxy_bypass_list is None:
             proxy_bypass_list = self.proxy_bypass_list
+        if proxy_pac_url is None:
+            proxy_pac_url = self.proxy_pac_url
         user_agent = agent
         if user_agent is None:
             user_agent = self.user_agent
@@ -3137,6 +3141,7 @@ def get_new_driver(
             port=port,
             proxy_string=proxy_string,
             proxy_bypass_list=proxy_bypass_list,
+            proxy_pac_url=proxy_pac_url,
             user_agent=user_agent,
             cap_file=cap_file,
             cap_string=cap_string,
@@ -12060,6 +12065,7 @@ def setUp(self, masterqa_mode=False):
             self.port = sb_config.port
             self.proxy_string = sb_config.proxy_string
             self.proxy_bypass_list = sb_config.proxy_bypass_list
+            self.proxy_pac_url = sb_config.proxy_pac_url
             self.user_agent = sb_config.user_agent
             self.mobile_emulator = sb_config.mobile_emulator
             self.device_metrics = sb_config.device_metrics
@@ -12372,6 +12378,7 @@ def setUp(self, masterqa_mode=False):
                 port=self.port,
                 proxy=self.proxy_string,
                 proxy_bypass_list=self.proxy_bypass_list,
+                proxy_pac_url=self.proxy_pac_url,
                 agent=self.user_agent,
                 switch_to=True,
                 cap_file=self.cap_file,