Merge pull request #678 from seleniumbase/grid-update-and-driver-install-update

Updates to Selenium Grid and WebDriver installation

Author: mdmintz

examples/example_logs/basic_test_info.txt

@@ -1,14 +1,22 @@
+examples.test_fail.MyTestClass.test_find_army_of_robots_on_xkcd_desert_island
+----------------------------------------------------
 Last Page: https://xkcd.com/731/
-  Browser: firefox
-Timestamp: 1540898481
+  Browser: firefox
+Timestamp: 1599068455  (Unix Timestamp)
+     Date: Wednesday, September 2, 2020
+     Time: 1:40:55 PM  (EDT, UTC-05:00)
+----------------------------------------------------
 Traceback: File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/unittest/case.py", line 615, in run
     testMethod()
-  File "/Users/michael/github/SeleniumBase/examples/test_fail.py", line 12, in test_find_army_of_robots_on_xkcd_desert_island
+  File "/Users/michael/github/SeleniumBase/examples/test_fail.py", line 17, in test_find_army_of_robots_on_xkcd_desert_island
     self.assert_element("div#ARMY_OF_ROBOTS", timeout=1)
-  File "/Users/michael/github/SeleniumBase/seleniumbase/fixtures/base_case.py", line 2166, in assert_element
+  File "/Users/michael/github/SeleniumBase/seleniumbase/fixtures/base_case.py", line 4931, in assert_element
     self.wait_for_element_visible(selector, by=by, timeout=timeout)
-  File "/Users/michael/github/SeleniumBase/seleniumbase/fixtures/base_case.py", line 2143, in wait_for_element_visible
+  File "/Users/michael/github/SeleniumBase/seleniumbase/fixtures/base_case.py", line 3290, in wait_for_element_visible
     self.driver, selector, by, timeout)
-  File "/Users/michael/github/SeleniumBase/seleniumbase/fixtures/page_actions.py", line 239, in wait_for_element_visible
-    selector, timeout, plural))
-Exception: Element {div#ARMY_OF_ROBOTS} was not visible after 1 second!
+  File "/Users/michael/github/SeleniumBase/seleniumbase/fixtures/page_actions.py", line 299, in wait_for_element_visible
+    timeout_exception(NoSuchElementException, message)
+  File "/Users/michael/github/SeleniumBase/seleniumbase/fixtures/page_actions.py", line 117, in timeout_exception
+    raise exc(message)
+Exception: Message: 
+ Element {div#ARMY_OF_ROBOTS} was not present after 1 second!

requirements.txt

@@ -1,7 +1,7 @@
 pip>=20.2.2
 packaging>=20.4
 setuptools>=44.1.1;python_version<"3.5"
-setuptools>=50.0.0;python_version>="3.5"
+setuptools>=50.0.3;python_version>="3.5"
 setuptools-scm>=4.1.2
 wheel>=0.35.1
 six==1.15.0
@@ -44,7 +44,7 @@ pyopenssl==19.1.0
 pygments==2.5.2;python_version<"3.5"
 pygments==2.6.1;python_version>="3.5"
 traitlets==4.3.3;python_version<"3.7"
-traitlets==5.0.0;python_version>="3.7"
+traitlets==5.0.2;python_version>="3.7"
 prompt-toolkit==1.0.18;python_version<"3.6"
 prompt-toolkit==3.0.7;python_version>="3.6"
 ipython==5.10.0;python_version<"3.5"

seleniumbase/console_scripts/run.py

@@ -382,11 +382,14 @@ def show_grid_hub_usage():
     print(sc)
     print("")
     print("  Usage:")
-    print("           seleniumbase grid-hub {start|stop}")
-    print("           OR:    sbase grid-hub {start|stop}")
+    print("           seleniumbase grid-hub {start|stop|restart} [OPTIONS]")
+    print("           OR:    sbase grid-hub {start|stop|restart} [OPTIONS]")
     print("  Options:")
     print("           -v, --verbose  (Increase verbosity of logging output.)")
     print("                          (Default: Quiet logging / not verbose.)")
+    print("           --timeout=TIMEOUT  (Close idle browser after TIMEOUT.)")
+    print("                              (The default TIMEOUT: 230 seconds.)")
+    print("                              (Use --timeout=0 to skip timeouts.)")
     print("  Example:")
     print("           seleniumbase grid-hub start")
     print("  Output:")
@@ -406,8 +409,8 @@ def show_grid_node_usage():
     print(sc)
     print("")
     print("  Usage:")
-    print("           seleniumbase grid-node {start|stop} [OPTIONS]")
-    print("           OR:    sbase grid-node {start|stop} [OPTIONS]")
+    print("           seleniumbase grid-node {start|stop|restart} [OPTIONS]")
+    print("           OR:    sbase grid-node {start|stop|restart} [OPTIONS]")
     print("  Options:")
     print("           --hub=[HUB_IP] (The Grid Hub IP Address to connect to.)")
     print("                          (Default: 127.0.0.1 if not set)")

seleniumbase/console_scripts/sb_install.py

@@ -41,9 +41,9 @@
 urllib3.disable_warnings()
 DRIVER_DIR = os.path.dirname(os.path.realpath(drivers.__file__))
 LOCAL_PATH = "/usr/local/bin/"  # On Mac and Linux systems
-DEFAULT_CHROMEDRIVER_VERSION = "2.44"
+DEFAULT_CHROMEDRIVER_VERSION = "2.44"  # (Specify "latest" to get the latest)
 DEFAULT_GECKODRIVER_VERSION = "v0.27.0"
-DEFAULT_EDGEDRIVER_VERSION = "85.0.564.44"
+DEFAULT_EDGEDRIVER_VERSION = "85.0.564.44"  # (Looks for LATEST_STABLE first)
 DEFAULT_OPERADRIVER_VERSION = "v.84.0.4147.89"
 
 
@@ -194,8 +194,6 @@ def main(override=None):
             print('#  %s: >>> %s' % (run_this, install_sb))
     elif name == "geckodriver" or name == "firefoxdriver":
         use_version = DEFAULT_GECKODRIVER_VERSION
-        if "win32" in sys_plat or "win64" in sys_plat or "x64" in sys_plat:
-            use_version = "v0.24.0"
         found_geckodriver = False
         if num_args == 4 or num_args == 5:
             if "-p" not in sys.argv[3].lower():
@@ -244,19 +242,33 @@ def main(override=None):
                             "version to download!\n")
     elif name == "edgedriver" or name == "msedgedriver":
         name = "edgedriver"
-        use_version = DEFAULT_EDGEDRIVER_VERSION
+        last = (
+            "https://msedgewebdriverstorage.blob.core.windows.net"
+            "/edgewebdriver/LATEST_STABLE")
+        get_latest = False
+        if num_args == 3:
+            get_latest = True
+        if num_args == 4 and "-p" in sys.argv[3].lower():
+            get_latest = True
         if num_args == 4 or num_args == 5:
             if "-p" not in sys.argv[3].lower():
                 use_version = sys.argv[3]
                 if use_version.lower() == "latest":
                     use_version = DEFAULT_EDGEDRIVER_VERSION
+                    get_latest = True
             else:
                 copy_to_path = True
         if num_args == 5:
             if "-p" in sys.argv[4].lower():
                 copy_to_path = True
             else:
                 invalid_run_command()
+        if get_latest:
+            url_request = requests.get(last)
+            if url_request.ok:
+                use_version = url_request.text.split('\r')[0].split('\n')[0]
+            else:
+                use_version = DEFAULT_EDGEDRIVER_VERSION
         if "win64" in sys_plat or "x64" in sys_plat:
             file_name = "edgedriver_win64.zip"
         elif "win32" in sys_plat or "x86" in sys_plat:
@@ -268,6 +280,11 @@ def main(override=None):
                             "only for Windows or Mac operating systems!")
         download_url = ("https://msedgedriver.azureedge.net/"
                         "%s/%s" % (use_version, file_name))
+        if not get_latest and not use_version == DEFAULT_EDGEDRIVER_VERSION:
+            url_request = requests.get(download_url)
+            if not url_request.ok:
+                raise Exception(
+                    "Could not find version [%s] of EdgeDriver!" % use_version)
         msg = c2 + "edgedriver version for download" + cr
         p_version = c3 + use_version + cr
         print("\n*** %s = %s" % (msg, p_version))

seleniumbase/core/browser_launcher.py

@@ -242,6 +242,8 @@ def _set_chrome_options(
     chrome_options.add_argument("--dom-automation")
     chrome_options.add_argument("--disable-hang-monitor")
     chrome_options.add_argument("--disable-prompt-on-repost")
+    if servername != "localhost":
+        use_auto_ext = True  # Use Automation Extension with the Selenium Grid
     if not use_auto_ext:  # (It's ON by default. Disable it when not wanted.)
         chrome_options.add_experimental_option("useAutomationExtension", False)
     if (settings.DISABLE_CSP_ON_CHROME or disable_csp) and not headless:

seleniumbase/utilities/selenium_grid/ReadMe.md

@@ -18,17 +18,18 @@ seleniumbase download server
 <h4><img src="https://seleniumbase.io/img/sb_icon.png" title="SeleniumBase" width="30" /> Grid Hub server controls:</h4>
 
 ```bash
-seleniumbase grid-hub {start|stop} [OPTIONS]
+seleniumbase grid-hub {start|stop|restart} [OPTIONS]
 ```
 <b>Options:</b>
 <ul>
 <li> -v / --verbose  (Increases verbosity of logging output.)</li>
+<li> --timeout=TIMEOUT  (Close idle browser after TIMEOUT sec.)</li>
 </ul>
 
 <h4><img src="https://seleniumbase.io/img/sb_icon.png" title="SeleniumBase" width="30" /> Grid node server controls:</h4>
 
 ```bash
-seleniumbase grid-node {start|stop} --hub=[HUB_IP] [OPTIONS]
+seleniumbase grid-node {start|stop|restart} --hub=[HUB_IP] [OPTIONS]
 ```
 <b>Options:</b>
 <ul>

seleniumbase/utilities/selenium_grid/grid-hub

@@ -1,16 +1,16 @@
 #!/usr/bin/env bash
 
 #
-# Usage: grid-hub {start|stop|restart}
+# Usage: grid-hub {start|stop} [TIMEOUT]
 #
 
 source $(dirname $0)/font_color
 
-EXPECTED_ARGS=1
+EXPECTED_ARGS=2
 E_BADARGS=65
 
 DO_showUsage() {
-    echo "Usage: $(basename $0) {start|stop|restart}"
+    echo "Usage: $(basename $0) {start|stop} [TIMEOUT]"
     exit $E_BADARGS
 }
 
@@ -34,7 +34,7 @@ if [ "$GRID_HUB_VERBOSE_LOGS" == "True" ]; then
 fi
 
 WEBDRIVER_SERVER_JAR=${DIR}/selenium-server-standalone.jar
-WEBDRIVER_HUB_PARAMS="-role hub -timeout 230 -browserTimeout 170 -port 4444"
+WEBDRIVER_HUB_PARAMS="-role hub -timeout $2 -browserTimeout 170 -port 4444"
 WEBDRIVER_HUB_PIDFILE="/tmp/webdriver_hub.pid"
 
 if [ ! -f $WEBDRIVER_SERVER_JAR ]; then

seleniumbase/utilities/selenium_grid/grid-node

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 #
-# Usage: grid-node {start|stop|restart}
+# Usage: grid-node {start|stop}
 #
 
 source $(dirname $0)/font_color
@@ -10,7 +10,7 @@ EXPECTED_ARGS=1
 E_BADARGS=65
 
 DO_showUsage() {
-    echo "Usage: $(basename $0) {start|stop|restart}"
+    echo "Usage: $(basename $0) {start|stop}"
     exit $E_BADARGS
 }
 

seleniumbase/utilities/selenium_grid/grid_hub.py

@@ -16,13 +16,16 @@
     os.environ["PATH"] = DRIVER_DIR + os.pathsep + os.environ["PATH"]
 
 
-def invalid_run_command():
+def invalid_run_command(msg=None):
     exp = ("  ** grid-hub **\n\n")
     exp += "  Usage:\n"
-    exp += "        seleniumbase grid-hub {start|stop|restart}\n"
+    exp += "        seleniumbase grid-hub {start|stop|restart} [OPTIONS]\n"
     exp += "  Options:\n"
     exp += "        -v, --verbose  (Increase verbosity of logging output.)\n"
     exp += "                       (Default: Quiet logging / not verbose.)\n"
+    exp += "        --timeout=TIMEOUT  (Close idle browser after TIMEOUT.)\n"
+    exp += "                           (The default TIMEOUT: 230 seconds.)\n"
+    exp += "                           (Use --timeout=0 to skip timeouts.)\n"
     exp += "  Example:\n"
     exp += "        seleniumbase grid-hub start\n"
     exp += "  Output:\n"
@@ -31,10 +34,13 @@ def invalid_run_command():
     exp += "        to speed up test runs and reduce the total time\n"
     exp += "        of test suite execution.\n"
     exp += "        You can start, restart, or stop the Grid Hub Server.\n"
+    if msg:
+        exp += msg
     raise Exception('INVALID RUN COMMAND!\n\n%s' % exp)
 
 
 def main():
+    timeout = 230  # The default number of seconds that a test can be idle
     dir_path = os.path.dirname(os.path.realpath(__file__))
     num_args = len(sys.argv)
     if sys.argv[0].split('/')[-1] == "seleniumbase" or (
@@ -55,6 +61,12 @@ def main():
         for option in options:
             if option == '-v' or option == '--verbose':
                 verbose = "True"
+            elif option.startswith("--timeout=") and len(option) > 10:
+                timeout = option.split("--timeout=")[1]
+                if not timeout.isdigit():
+                    msg = '\n"timeout" must be a non-negative integer!\n'
+                    print(msg)
+                    invalid_run_command(msg)
             else:
                 invalid_run_command()
 
@@ -70,18 +82,22 @@ def main():
 
     if "linux" in sys.platform or "darwin" in sys.platform:
         if grid_hub_command == "start":
-            subprocess.check_call(dir_path + "/grid-hub start", shell=True)
+            subprocess.check_call(
+                dir_path + "/grid-hub start %s" % timeout, shell=True)
         elif grid_hub_command == "restart":
-            subprocess.check_call(dir_path + "/grid-hub restart", shell=True)
+            subprocess.check_call(dir_path + "/grid-hub stop .", shell=True)
+            subprocess.check_call(
+                dir_path + "/grid-hub start %s" % timeout, shell=True)
         elif grid_hub_command == "stop":
-            subprocess.check_call(dir_path + "/grid-hub stop", shell=True)
+            subprocess.check_call(dir_path + "/grid-hub stop .", shell=True)
         else:
             invalid_run_command()
     else:
         if grid_hub_command == "start" or grid_hub_command == "restart":
             shell_command = (
                 """java -jar %s/selenium-server-standalone.jar -role hub """
-                """-timeout 230 -browserTimeout 170 -port 4444""" % dir_path)
+                """-timeout %s -browserTimeout 170 -port 4444"""
+                "" % (dir_path, timeout))
             print("\nStarting Selenium-WebDriver Grid Hub...\n")
             print(shell_command)
             print("")

seleniumbase/utilities/selenium_grid/grid_node.py

@@ -99,7 +99,8 @@ def main():
         if grid_hub_command == "start":
             subprocess.check_call(dir_path + "/grid-node start", shell=True)
         elif grid_hub_command == "restart":
-            subprocess.check_call(dir_path + "/grid-node restart", shell=True)
+            subprocess.check_call(dir_path + "/grid-node stop", shell=True)
+            subprocess.check_call(dir_path + "/grid-node start", shell=True)
         elif grid_hub_command == "stop":
             subprocess.check_call(dir_path + "/grid-node stop", shell=True)
         else: