Merge pull request #546 from seleniumbase/fix-chrome-proxy-with-auth

Fix Chrome proxy with auth; and a few other updates

Author: mdmintz

README.md

@@ -242,6 +242,7 @@ SeleniumBase provides additional Pytest command-line options for tests:
 --verify-delay=SECONDS  # (The delay before MasterQA verification checks.)
 --disable-csp  # (This disables the Content Security Policy of websites.)
 --enable-sync  # (The option to enable "Chrome Sync".)
+--use-auto-ext  # (The option to use Chrome's automation extension.)
 --no-sandbox  # (The option to enable Chrome's "No-Sandbox" feature.)
 --disable-gpu  # (The option to enable Chrome's "Disable GPU" feature.)
 --incognito  #  (The option to enable Chrome's Incognito mode.)

examples/capabilities/ReadMe.md

@@ -50,7 +50,14 @@ A regex parser was built into SeleniumBase to capture all lines from the specifi
 ``caps['KEY'] = False``
 (Each pair must be on a separate line. You can interchange single and double quotes.)
 
-You can also swap ``--browser=remote`` with an actual browser, eg ``--browser=chrome``, which will combine the default SeleniumBase desired capabilities with those that were specified in the capabilities file when using ``--cap_file=FILE.py``. Capabilities will override other parameters, so if you set the browser to one thing and the capabilities browser to another, SeleniumBase will use the capabilities browser as the browser. You'll need default SeleniumBase desired capabilities when using a proxy server (not the same as a Selenium Grid server), when downloading files to a desired folder, for disabling some warnings on Chrome, for overriding a website's Content Security Policy on Firefox, and for other reasons.
+You can also swap ``--browser=remote`` with an actual browser, eg ``--browser=chrome``, which will combine the default SeleniumBase desired capabilities with those that were specified in the capabilities file when using ``--cap_file=FILE.py``. Capabilities will override other parameters, so if you set the browser to one thing and the capabilities browser to another, SeleniumBase will use the capabilities browser as the browser.
+
+You'll need default SeleniumBase capabilities for:
+* Using a proxy server (not the same as a Selenium Grid server)
+* Downloading files to a desired folder
+* Disabling some warnings on Chrome
+* Overriding a website's Content Security Policy on Chrome
+* Other possible reasons
 
 You can also set browser desired capabilities from a command line string:
 Example:
@@ -59,8 +66,17 @@ pytest test_swag_labs.py --cap-string='{"browserName":"chrome","name":"test1"}'
 ```
 (Enclose cap-string in single quotes. Enclose parameter keys in double quotes.)
 
-If using a local Selenium Grid with SeleniumBase, make sure to start up the Grid Hub and nodes first:
+If you pass ``"*"`` into the ``"name"`` field of ``--cap-string``, the name will become the test identifier. Example:
+```bash
+pytest test_swag_labs.py --cap-string='{"browserName":"chrome","name":"*"}' --server="127.0.0.1" --browser=chrome
+```
+Example name: ``"my_first_test.MyTestClass.test_basic"``
+
+### Using a local Selenium Grid
+
+If using a local Selenium Grid with SeleniumBase, start up the Grid Hub and nodes first:
 ```bash
 seleniumbase grid-hub start
 seleniumbase grid-node start
 ```
+(The Selenium Server JAR file will be automatically downloaded for first-time Grid users. You'll also need Java installed to start up the Grid.)

examples/raw_parameter_script.py

@@ -47,6 +47,7 @@
     sb.archive_logs = False
     sb.disable_csp = False
     sb.enable_sync = False
+    sb.use_auto_ext = False
     sb.no_sandbox = False
     sb.disable_gpu = False
     sb._reuse_session = False

help_docs/customizing_test_runs.md

@@ -115,6 +115,7 @@ SeleniumBase provides additional Pytest command-line options for tests:
 --verify-delay=SECONDS  # (The delay before MasterQA verification checks.)
 --disable-csp  # (This disables the Content Security Policy of websites.)
 --enable-sync  # (The option to enable "Chrome Sync".)
+--use-auto-ext  # (The option to use Chrome's automation extension.)
 --no-sandbox  # (The option to enable Chrome's "No-Sandbox" feature.)
 --disable-gpu  # (The option to enable Chrome's "Disable GPU" feature.)
 --incognito  #  (The option to enable Chrome's Incognito mode.)

requirements.txt

@@ -29,7 +29,6 @@ soupsieve==1.9.5;python_version<"3.5"
 soupsieve==2.0;python_version>="3.5"
 beautifulsoup4==4.9.0
 atomicwrites==1.3.0
-portalocker==1.7.0
 cryptography==2.9
 pyopenssl==19.1.0
 pygments==2.5.2;python_version<"3.5"

seleniumbase/config/proxy_list.py

@@ -23,7 +23,6 @@
     "example1": "142.93.130.169:8118",  # (Example) - set your own proxy here
     "example2": "51.91.212.159:3128",  # (Example) - set your own proxy here
     "example3": "109.74.142.138:53281",  # (Example) - set your own proxy here
-    "example4": "91.207.60.241:1282",  # (Example) - set your own proxy here
     "proxy1": None,
     "proxy2": None,
     "proxy3": None,

seleniumbase/console_scripts/run.py

@@ -53,9 +53,9 @@ def show_basic_usage():
     print("       inject-objects [SELENIUMBASE_PYTHON_FILE] [OPTIONS]")
     print("       objectify [SELENIUMBASE_PYTHON_FILE] [OPTIONS]")
     print("       revert-objects [SELENIUMBASE_PYTHON_FILE]")
-    print("       encrypt OR obfuscate")
-    print("       decrypt OR unobfuscate")
-    print("       download server")
+    print("       encrypt   (OR: obfuscate)")
+    print("       decrypt   (OR: unobfuscate)")
+    print("       download server   (The Selenium Server JAR file)")
     print("       grid-hub [start|stop|restart] [OPTIONS]")
     print("       grid-node [start|stop|restart] --hub=[HUB_IP] [OPTIONS]")
     print('  * (EXAMPLE: "seleniumbase install chromedriver") *')
@@ -259,7 +259,12 @@ def show_grid_node_usage():
 
 def get_version():
     import pkg_resources
-    return pkg_resources.require("seleniumbase")[0:1]
+    version_info = None
+    try:
+        version_info = pkg_resources.require("seleniumbase")[0:1]
+    except Exception:
+        version_info = ["ERROR: Cannot detect version! Please reinstall!"]
+    return version_info
 
 
 def show_version_info():

seleniumbase/core/browser_launcher.py

@@ -132,8 +132,8 @@ def _add_chrome_disable_csp_extension(chrome_options):
 def _set_chrome_options(
         downloads_path, headless,
         proxy_string, proxy_auth, proxy_user, proxy_pass,
-        user_agent, disable_csp, enable_sync, no_sandbox, disable_gpu,
-        incognito, guest_mode, devtools,
+        user_agent, disable_csp, enable_sync, use_auto_ext,
+        no_sandbox, disable_gpu, incognito, guest_mode, devtools,
         user_data_dir, extension_zip, extension_dir, servername,
         mobile_emulator, device_width, device_height, device_pixel_ratio):
     chrome_options = webdriver.ChromeOptions()
@@ -211,15 +211,12 @@ def _set_chrome_options(
     chrome_options.add_argument("--disable-web-security")
     chrome_options.add_argument("--homepage=about:blank")
     chrome_options.add_argument("--dom-automation")
-    if servername == "localhost" or servername == "127.0.0.1":
+    if not use_auto_ext:  # (It's ON by default. Disable it when not wanted.)
         chrome_options.add_experimental_option("useAutomationExtension", False)
     if (settings.DISABLE_CSP_ON_CHROME or disable_csp) and not headless:
         # Headless Chrome doesn't support extensions, which are required
         # for disabling the Content Security Policy on Chrome
         chrome_options = _add_chrome_disable_csp_extension(chrome_options)
-    elif not extension_zip and not extension_dir:
-        if servername == "localhost" or servername == "127.0.0.1":
-            chrome_options.add_argument("--disable-extensions")
     if proxy_string:
         if proxy_auth:
             chrome_options = _add_chrome_proxy_extension(
@@ -351,11 +348,11 @@ def validate_proxy_string(proxy_string):
 def get_driver(browser_name, headless=False, use_grid=False,
                servername='localhost', port=4444, proxy_string=None,
                user_agent=None, cap_file=None, cap_string=None,
-               disable_csp=None, enable_sync=None,
+               disable_csp=None, enable_sync=None, use_auto_ext=None,
                no_sandbox=None, disable_gpu=None,
                incognito=None, guest_mode=None, devtools=None,
                user_data_dir=None, extension_zip=None, extension_dir=None,
-               mobile_emulator=False, device_width=None,
+               test_id=None, mobile_emulator=False, device_width=None,
                device_height=None, device_pixel_ratio=None):
     proxy_auth = False
     proxy_user = None
@@ -390,15 +387,15 @@ def get_driver(browser_name, headless=False, use_grid=False,
         return get_remote_driver(
             browser_name, headless, servername, port,
             proxy_string, proxy_auth, proxy_user, proxy_pass, user_agent,
-            cap_file, cap_string, disable_csp, enable_sync,
+            cap_file, cap_string, disable_csp, enable_sync, use_auto_ext,
             no_sandbox, disable_gpu, incognito, guest_mode, devtools,
-            user_data_dir, extension_zip, extension_dir,
+            user_data_dir, extension_zip, extension_dir, test_id,
             mobile_emulator, device_width, device_height, device_pixel_ratio)
     else:
         return get_local_driver(
             browser_name, headless, servername,
             proxy_string, proxy_auth, proxy_user, proxy_pass, user_agent,
-            disable_csp, enable_sync, no_sandbox, disable_gpu,
+            disable_csp, enable_sync, use_auto_ext, no_sandbox, disable_gpu,
             incognito, guest_mode, devtools,
             user_data_dir, extension_zip, extension_dir,
             mobile_emulator, device_width, device_height, device_pixel_ratio)
@@ -407,9 +404,9 @@ def get_driver(browser_name, headless=False, use_grid=False,
 def get_remote_driver(
         browser_name, headless, servername, port, proxy_string, proxy_auth,
         proxy_user, proxy_pass, user_agent, cap_file, cap_string,
-        disable_csp, enable_sync, no_sandbox, disable_gpu,
+        disable_csp, enable_sync, use_auto_ext, no_sandbox, disable_gpu,
         incognito, guest_mode, devtools,
-        user_data_dir, extension_zip, extension_dir,
+        user_data_dir, extension_zip, extension_dir, test_id,
         mobile_emulator, device_width, device_height, device_pixel_ratio):
     downloads_path = download_helper.get_downloads_folder()
     download_helper.reset_downloads_folder()
@@ -424,17 +421,21 @@ def get_remote_driver(
         except Exception as e:
             p1 = "Invalid input format for --cap-string:\n  %s" % e
             p2 = "The --cap-string input was: %s" % cap_string
-            p3 = "Enclose cap-string in single quotes; keys in double quotes."
+            p3 = "Enclose cap-string in SINGLE quotes; keys in DOUBLE quotes."
             p4 = ("""Here's an example of correct cap-string usage:\n  """
                   """--cap-string='{"browserName":"chrome","name":"test1"}'""")
             raise Exception("%s\n%s\n%s\n%s" % (p1, p2, p3, p4))
         for cap_key in extra_caps.keys():
             desired_caps[cap_key] = extra_caps[cap_key]
+    if cap_file or cap_string:
+        if "name" in desired_caps.keys():
+            if desired_caps["name"] == "*":
+                desired_caps["name"] = test_id
     if browser_name == constants.Browser.GOOGLE_CHROME:
         chrome_options = _set_chrome_options(
             downloads_path, headless,
             proxy_string, proxy_auth, proxy_user, proxy_pass, user_agent,
-            disable_csp, enable_sync, no_sandbox, disable_gpu,
+            disable_csp, enable_sync, use_auto_ext, no_sandbox, disable_gpu,
             incognito, guest_mode, devtools,
             user_data_dir, extension_zip, extension_dir, servername,
             mobile_emulator, device_width, device_height, device_pixel_ratio)
@@ -546,7 +547,7 @@ def get_remote_driver(
 def get_local_driver(
         browser_name, headless, servername,
         proxy_string, proxy_auth, proxy_user, proxy_pass, user_agent,
-        disable_csp, enable_sync, no_sandbox, disable_gpu,
+        disable_csp, enable_sync, use_auto_ext, no_sandbox, disable_gpu,
         incognito, guest_mode, devtools,
         user_data_dir, extension_zip, extension_dir,
         mobile_emulator, device_width, device_height, device_pixel_ratio):
@@ -636,8 +637,8 @@ def get_local_driver(
             chrome_options = _set_chrome_options(
                 downloads_path, headless,
                 proxy_string, proxy_auth, proxy_user, proxy_pass, user_agent,
-                disable_csp, enable_sync, no_sandbox, disable_gpu,
-                incognito, guest_mode, devtools,
+                disable_csp, enable_sync, use_auto_ext,
+                no_sandbox, disable_gpu, incognito, guest_mode, devtools,
                 user_data_dir, extension_zip, extension_dir, servername,
                 mobile_emulator, device_width, device_height,
                 device_pixel_ratio)
@@ -692,8 +693,8 @@ def get_local_driver(
             chrome_options = _set_chrome_options(
                 downloads_path, headless,
                 proxy_string, proxy_auth, proxy_user, proxy_pass, user_agent,
-                disable_csp, enable_sync, no_sandbox, disable_gpu,
-                incognito, guest_mode, devtools,
+                disable_csp, enable_sync, use_auto_ext,
+                no_sandbox, disable_gpu, incognito, guest_mode, devtools,
                 user_data_dir, extension_zip, extension_dir, servername,
                 mobile_emulator, device_width, device_height,
                 device_pixel_ratio)

seleniumbase/fixtures/base_case.py

@@ -1522,7 +1522,7 @@ def switch_to_default_window(self):
     def get_new_driver(self, browser=None, headless=None,
                        servername=None, port=None, proxy=None, agent=None,
                        switch_to=True, cap_file=None, cap_string=None,
-                       disable_csp=None, enable_sync=None,
+                       disable_csp=None, enable_sync=None, use_auto_ext=None,
                        no_sandbox=None, disable_gpu=None,
                        incognito=None, guest_mode=None, devtools=None,
                        user_data_dir=None, extension_zip=None,
@@ -1543,6 +1543,7 @@ def get_new_driver(self, browser=None, headless=None,
             cap_string - the string with desired capabilities for the browser
             disable_csp - an option to disable Chrome's Content Security Policy
             enable_sync - the option to enable the Chrome Sync feature (Chrome)
+            use_auto_ext - the option to enable Chrome's Automation Extension
             no_sandbox - the option to enable the "No-Sandbox" feature (Chrome)
             disable_gpu - the option to enable Chrome's "Disable GPU" feature
             incognito - the option to enable Chrome's Incognito mode (Chrome)
@@ -1600,6 +1601,8 @@ def get_new_driver(self, browser=None, headless=None,
             disable_csp = self.disable_csp
         if enable_sync is None:
             enable_sync = self.enable_sync
+        if use_auto_ext is None:
+            use_auto_ext = self.use_auto_ext
         if no_sandbox is None:
             no_sandbox = self.no_sandbox
         if disable_gpu is None:
@@ -1619,6 +1622,7 @@ def get_new_driver(self, browser=None, headless=None,
         # Due to https://stackoverflow.com/questions/23055651/ , skip extension
         # if self.demo_mode or self.masterqa_mode:
         #    disable_csp = True
+        test_id = self.__get_test_id()
         if cap_file is None:
             cap_file = self.cap_file
         if cap_string is None:
@@ -1648,6 +1652,7 @@ def get_new_driver(self, browser=None, headless=None,
                                                  cap_string=cap_string,
                                                  disable_csp=disable_csp,
                                                  enable_sync=enable_sync,
+                                                 use_auto_ext=use_auto_ext,
                                                  no_sandbox=no_sandbox,
                                                  disable_gpu=disable_gpu,
                                                  incognito=incognito,
@@ -1656,6 +1661,7 @@ def get_new_driver(self, browser=None, headless=None,
                                                  user_data_dir=user_data_dir,
                                                  extension_zip=extension_zip,
                                                  extension_dir=extension_dir,
+                                                 test_id=test_id,
                                                  mobile_emulator=is_mobile,
                                                  device_width=d_width,
                                                  device_height=d_height,
@@ -4547,6 +4553,7 @@ def setUp(self, masterqa_mode=False):
             self.verify_delay = sb_config.verify_delay
             self.disable_csp = sb_config.disable_csp
             self.enable_sync = sb_config.enable_sync
+            self.use_auto_ext = sb_config.use_auto_ext
             self.no_sandbox = sb_config.no_sandbox
             self.disable_gpu = sb_config.disable_gpu
             self.incognito = sb_config.incognito
@@ -4705,6 +4712,7 @@ def setUp(self, masterqa_mode=False):
                                               cap_string=self.cap_string,
                                               disable_csp=self.disable_csp,
                                               enable_sync=self.enable_sync,
+                                              use_auto_ext=self.use_auto_ext,
                                               no_sandbox=self.no_sandbox,
                                               disable_gpu=self.disable_gpu,
                                               incognito=self.incognito,

seleniumbase/plugins/pytest_plugin.py

@@ -45,6 +45,7 @@ def pytest_addoption(parser):
     --verify-delay=SECONDS  (The delay before MasterQA verification checks.)
     --disable-csp  (This disables the Content Security Policy of websites.)
     --enable-sync  (The option to enable "Chrome Sync".)
+    --use-auto-ext  (The option to use Chrome's automation extension.)
     --no-sandbox  (The option to enable Chrome's "No-Sandbox" feature.)
     --disable-gpu  (The option to enable Chrome's "Disable GPU" feature.)
     --incognito  (The option to enable Chrome's Incognito mode.)
@@ -350,6 +351,13 @@ def pytest_addoption(parser):
                      dest='enable_sync',
                      default=False,
                      help="""Using this enables the "Chrome Sync" feature.""")
+    parser.addoption('--use_auto_ext', '--use-auto-ext', '--auto-ext',
+                     action="store_true",
+                     dest='use_auto_ext',
+                     default=False,
+                     help="""Using this enables Chrome's Automation Extension.
+                          It's not required, but some commands & advanced
+                          features may need it.""")
     parser.addoption('--no_sandbox', '--no-sandbox',
                      action="store_true",
                      dest='no_sandbox',
@@ -462,6 +470,7 @@ def pytest_configure(config):
     sb_config.verify_delay = config.getoption('verify_delay')
     sb_config.disable_csp = config.getoption('disable_csp')
     sb_config.enable_sync = config.getoption('enable_sync')
+    sb_config.use_auto_ext = config.getoption('use_auto_ext')
     sb_config.no_sandbox = config.getoption('no_sandbox')
     sb_config.disable_gpu = config.getoption('disable_gpu')
     sb_config.incognito = config.getoption('incognito')