Merge pull request #1084 from seleniumbase/shadow-dom-overhaul-and-more

mdmintz · web-flow · commit d570ae8454f2 · 2021-11-25T13:13:03.000-05:00
Shadow DOM overhaul and more
diff --git a/examples/desktop_apps/ReadMe.md b/examples/desktop_apps/ReadMe.md
@@ -0,0 +1,5 @@
+<h3 align="left"><img src="https://seleniumbase.io/cdn/img/sb_logo_b.png" alt="SeleniumBase" width="320" /></h3>
+
+<h2><img src="https://seleniumbase.io/img/logo6.png" title="SeleniumBase" width="32" /> Desktop Apps</h2>
+
+* **Recorder** (Run using ``python recorder.py``)
diff --git a/examples/desktop_apps/recorder.py b/examples/desktop_apps/recorder.py
@@ -0,0 +1,63 @@
+""" Run this file using `python recorder.py` """
+
+import os
+import sys
+from seleniumbase.fixtures import page_utils
+
+if sys.version_info[0] < 3:
+    raise Exception("This script is for Python 3 only!")
+import tkinter as tk  # noqa: E402
+from tkinter import messagebox  # noqa: E402
+
+
+def do_recording(file_name, url):
+    url = url.strip()
+    if not page_utils.is_valid_url(url):
+        if page_utils.is_valid_url("https://" + url):
+            url = "https://" + url
+    if not page_utils.is_valid_url(url):
+        messagebox.showwarning(
+            "Invalid URL", "Enter a valid URL. (Eg. https://google.com)")
+    else:
+        if os.path.exists(file_name):
+            os.remove(file_name)
+        os.system("python -m sbase mkrec %s --url=%s" % (file_name, url))
+
+
+def do_playback(file_name):
+    os.system("pytest %s --verbose --capture=no" % file_name)
+
+
+def create_tkinter_gui(file_name):
+    window = tk.Tk()
+    window.title("Recorder App")
+    window.geometry("360x175")
+    frame = tk.Frame(window)
+    frame.pack()
+
+    a = tk.StringVar()
+    tk.Label(window, text="Enter URL to start recording on:").pack()
+    entry = tk.Entry(window, textvariable=a)
+    entry.pack()
+    entry.focus()
+    entry.bind("<Return>", (lambda _: do_recording(file_name, a.get())))
+    tk.Button(
+        window, text="Record", command=lambda: do_recording(file_name, a.get())
+    ).pack()
+    tk.Label(window, text="").pack()
+    tk.Label(
+        window, text="Playback the latest recording:").pack()
+    tk.Button(
+        window, text="Playback", command=lambda: do_playback(file_name)
+    ).pack()
+
+    # Bring form to front
+    window.lift()
+    window.attributes("-topmost", True)
+    window.after_idle(window.attributes, "-topmost", False)
+    window.mainloop()
+
+
+if __name__ == "__main__":
+    file_name = "new_recording.py"
+    create_tkinter_gui(file_name)
diff --git a/examples/shadow_root_test.py b/examples/shadow_root_test.py
@@ -1,14 +1,19 @@
+""" Piercing through shadow-root elements with the "::shadow" selector.
+    To confirm that "::shadow" works, print text and assert exact text. """
+
 from seleniumbase import BaseCase
 
 
 class ShadowRootTest(BaseCase):
     def test_shadow_root(self):
-        self.open("https://react-shadow.herokuapp.com/Patagonia")
-        self.click("section.weather::shadow div::shadow button")
-        self.assert_element('section.weather::shadow img[alt="Patagonia"]')
-        weather = self.get_text("section.weather::shadow h1")
-        self.post_message(weather)
-        self.click('section.weather::shadow a[href="/Kyoto"]')
-        self.assert_element('section.weather::shadow img[alt="Kyoto"]')
-        weather = self.get_text("section.weather::shadow h1")
-        self.post_message(weather)
+        self.open("https://seleniumbase.io/other/shadow_dom")
+        print("")
+        self.click("button.tab_1")
+        print(self.get_text("fancy-tabs::shadow #panels"))
+        self.assert_exact_text("Content Panel 1", "fancy-tabs::shadow #panels")
+        self.click("button.tab_2")
+        print(self.get_text("fancy-tabs::shadow #panels"))
+        self.assert_exact_text("Content Panel 2", "fancy-tabs::shadow #panels")
+        self.click("button.tab_3")
+        print(self.get_text("fancy-tabs::shadow #panels"))
+        self.assert_exact_text("Content Panel 3", "fancy-tabs::shadow #panels")
diff --git a/examples/test_decryption.py b/examples/test_decryption.py
@@ -1,6 +1,6 @@
 """
-This test demonstrates the use of encryption/decryption.
-(Technically, obfuscation/unobfuscation of passwords.)
+This test demonstrates the use of password encryption/decryption.
+(Technically considered to be obfuscation/unobfuscation.)
 """
 
 from seleniumbase import BaseCase
diff --git a/help_docs/ReadMe.md b/help_docs/ReadMe.md
@@ -78,14 +78,16 @@
 <div><a href="https://seleniumbase.io/tinymce/"><b>TinyMCE (Test Page)</b></a></div>
 <div><a href="https://seleniumbase.io/error_page/"><b>Error Page (Test Page)</b></a></div>
 <div><a href="https://seleniumbase.io/other/drag_and_drop"><b>Drag-&-Drop (Test Page)</b></a></div>
-<div><a href="https://seleniumbase.io/devices/"><b>Virtual Device Farm</b></a></div>
+<div><a href="https://seleniumbase.io/devices/"><b>Device Farm (Virtual)</b></a></div>
 <div><a href="https://seleniumbase.io/w3schools/"><b>HTML Playground Page</b></a></div>
 <div><a href="https://seleniumbase.io/w3schools/sbase"><b>SeleniumBase in iframe</b></a></div>
 <div><a href="https://seleniumbase.io/other/broken_page"><b>Page with broken links</b></a></div>
+<div><a href="https://seleniumbase.io/other/shadow_dom"><b>Shadow DOM/Root</b></a></div>
 <div><a href="https://seleniumbase.io/w3schools/iframes"><b>W3Schools iframes</b></a></div>
+<div><a href="https://seleniumbase.io/w3schools/file_upload"><b>W3Schools file upload</b></a></div>
 <div><a href="https://seleniumbase.io/w3schools/double_click"><b>W3Schools doubleclick</b></a></div>
-<div><a href="https://seleniumbase.io/w3schools/checkboxes"><b>W3Schools checkboxes</b></a></div>
 <div><a href="https://seleniumbase.io/w3schools/drag_drop"><b>W3Schools drag & drop</b></a></div>
+<div><a href="https://seleniumbase.io/w3schools/checkboxes"><b>W3Schools checkboxes</b></a></div>
 <div><a href="https://seleniumbase.io/w3schools/radio_buttons"><b>W3Schools radio buttons</b></a></div>
 
 --------
diff --git a/help_docs/method_summary.md b/help_docs/method_summary.md
@@ -396,6 +396,8 @@ self.assert_no_js_errors()
 
 self.inspect_html()
 
+self.is_valid_url(url)
+
 self.is_chromium()
 
 self.get_chrome_version()
diff --git a/mkdocs.yml b/mkdocs.yml
@@ -128,14 +128,16 @@ nav:
     - TinyMCE (Test Page): https://seleniumbase.io/tinymce/
     - Error Page (Test Page): https://seleniumbase.io/error_page/
     - Drag-&-Drop (Test Page): https://seleniumbase.io/other/drag_and_drop
-    - Virtual Device Farm: https://seleniumbase.io/devices/
+    - Device Farm (Virtual): https://seleniumbase.io/devices/
     - HTML Playground Page: https://seleniumbase.io/w3schools/
     - SeleniumBase in iframe: https://seleniumbase.io/w3schools/sbase
     - Page with broken links: https://seleniumbase.io/other/broken_page
+    - Shadow DOM/Root: https://seleniumbase.io/other/shadow_dom
     - W3Schools iframes: https://seleniumbase.io/w3schools/iframes
+    - W3Schools file upload: https://seleniumbase.io/w3schools/file_upload
     - W3Schools doubleclick: https://seleniumbase.io/w3schools/double_click
-    - W3Schools checkboxes: https://seleniumbase.io/w3schools/checkboxes
     - W3Schools drag & drop: https://seleniumbase.io/w3schools/drag_drop
+    - W3Schools checkboxes: https://seleniumbase.io/w3schools/checkboxes
     - W3Schools radio buttons: https://seleniumbase.io/w3schools/radio_buttons
   - Presentations:
     - Presenter Demo: https://seleniumbase.io/other/presenter.html
diff --git a/mkdocs_build/requirements.txt b/mkdocs_build/requirements.txt
@@ -1,14 +1,14 @@
 regex>=2021.11.10
 tqdm>=4.62.3
-docutils==0.18
+docutils==0.18.1
 python-dateutil==2.8.2
 livereload==2.6.3;python_version>="3.6"
 joblib==1.1.0;python_version>="3.6"
 Markdown==3.3.6;python_version>="3.6"
 MarkupSafe==2.0.1;python_version>="3.6"
-pyparsing==2.4.7;python_version>="3.6"
-keyring==23.2.1;python_version>="3.6"
-pkginfo==1.7.1;python_version>="3.6"
+pyparsing==3.0.6;python_version>="3.6"
+keyring==23.3.0;python_version>="3.6"
+pkginfo==1.8.1;python_version>="3.6"
 Jinja2==3.0.3;python_version>="3.6"
 click==8.0.3;python_version>="3.6"
 zipp==3.6.0;python_version>="3.6"
diff --git a/requirements.txt b/requirements.txt
@@ -1,10 +1,10 @@
 pip>=20.3.4;python_version<"3.6"
 pip>=21.3.1;python_version>="3.6"
 packaging>=20.9;python_version<"3.6"
-packaging>=21.2;python_version>="3.6"
+packaging>=21.3;python_version>="3.6"
 setuptools>=44.1.1;python_version<"3.5"
 setuptools>=50.3.2;python_version>="3.5" and python_version<"3.6"
-setuptools>=59.1.1;python_version>="3.6"
+setuptools>=59.2.0;python_version>="3.6"
 setuptools-scm>=5.0.2;python_version<"3.6"
 setuptools-scm>=6.3.2;python_version>="3.6"
 tomli>=1.2.2;python_version>="3.6"
@@ -25,7 +25,7 @@ idna==2.10;python_version<"3.6"
 idna==3.3;python_version>="3.6"
 chardet==3.0.4;python_version<"3.5"
 chardet==4.0.0;python_version>="3.5"
-charset-normalizer==2.0.7;python_version>="3.5"
+charset-normalizer==2.0.8;python_version>="3.5"
 urllib3==1.26.7
 requests==2.26.0;python_version<"3.5"
 requests==2.25.1;python_version>="3.5" and python_version<"3.6"
@@ -37,10 +37,10 @@ trio-websocket==0.9.2;python_version>="3.7"
 pyopenssl==21.0.0;python_version>="3.7"
 wsproto==1.0.0;python_version>="3.7"
 selenium==3.141.0;python_version<"3.7"
-selenium==4.0.0;python_version>="3.7"
+selenium==4.1.0;python_version>="3.7"
 msedge-selenium-tools==3.141.3;python_version<"3.7"
 more-itertools==5.0.0;python_version<"3.5"
-more-itertools==8.11.0;python_version>="3.5"
+more-itertools==8.12.0;python_version>="3.5"
 cssselect==1.1.0
 sortedcontainers==2.4.0
 filelock==3.2.1;python_version<"3.6"
@@ -77,7 +77,7 @@ beautifulsoup4==4.10.0;python_version>="3.5"
 cryptography==2.9.2;python_version<"3.5"
 cryptography==3.2.1;python_version>="3.5" and python_version<"3.6"
 cryptography==3.4.8;python_version>="3.6" and python_version<"3.7"
-cryptography==35.0.0;python_version>="3.7"
+cryptography==36.0.0;python_version>="3.7"
 pygments==2.5.2;python_version<"3.5"
 pygments==2.10.0;python_version>="3.5"
 prompt-toolkit==1.0.18;python_version<"3.5"
@@ -107,7 +107,8 @@ toml==0.10.2
 Pillow==6.2.2;python_version<"3.5"
 Pillow==7.2.0;python_version>="3.5" and python_version<"3.6"
 Pillow==8.4.0;python_version>="3.6"
-typing-extensions==3.10.0.2;python_version<"3.8"
+typing-extensions==3.10.0.2;python_version<"3.6"
+typing-extensions==4.0.0;python_version>="3.6" and python_version<"3.8"
 rich==10.14.0;python_version>="3.6" and python_version<"4.0"
 tornado==5.1.1;python_version<"3.5"
 tornado==6.1;python_version>="3.5"
diff --git a/seleniumbase/__version__.py b/seleniumbase/__version__.py
@@ -1,2 +1,2 @@
 # seleniumbase package
-__version__ = "2.1.9"
+__version__ = "2.2.0"
diff --git a/seleniumbase/core/browser_launcher.py b/seleniumbase/core/browser_launcher.py
@@ -508,7 +508,7 @@ def _set_firefox_options(
     options.set_preference("devtools.errorconsole.enabled", True)
     options.set_preference("dom.webnotifications.enabled", False)
     options.set_preference("dom.disable_beforeunload", True)
-    options.set_preference("browser.contentblocking.database.enabled", False)
+    options.set_preference("browser.contentblocking.database.enabled", True)
     options.set_preference("extensions.allowPrivateBrowsingByDefault", True)
     options.set_preference("extensions.PrivateBrowsing.notification", False)
     options.set_preference("extensions.systemAddon.update.enabled", False)
diff --git a/seleniumbase/fixtures/base_case.py b/seleniumbase/fixtures/base_case.py
@@ -68,6 +68,9 @@ def test_anything(self):
 if sys.version_info[0] < 3:
     reload(sys)  # noqa: F821
     sys.setdefaultencoding("utf8")
+selenium4 = False
+if sys.version_info[0] == 3 and sys.version_info[1] >= 7:
+    selenium4 = True
 
 
 class BaseCase(unittest.TestCase):
@@ -5241,6 +5244,10 @@ def inspect_html(self):
         print(results)
         return results
 
+    def is_valid_url(self, url):
+        """ Return True if the url is a valid url. """
+        return page_utils.is_valid_url(url)
+
     def is_chromium(self):
         """ Return True if the browser is Chrome, Edge, or Opera. """
         self.__check_scope()
@@ -5736,9 +5743,48 @@ def __get_shadow_element(self, selector, timeout=None):
         element = self.get_element(selectors[0])
         selector_chain = selectors[0]
         for selector_part in selectors[1:]:
-            shadow_root = self.execute_script(
-                "return arguments[0].shadowRoot", element
-            )
+            shadow_root = None
+            if (
+                selenium4
+                and self.is_chromium()
+                and int(self.__get_major_browser_version()) >= 96
+            ):
+                try:
+                    shadow_root = element.shadow_root
+                except Exception:
+                    if self.browser == "chrome":
+                        chrome_dict = self.driver.capabilities["chrome"]
+                        chrome_dr_version = chrome_dict["chromedriverVersion"]
+                        chromedriver_version = chrome_dr_version.split(" ")[0]
+                        major_c_dr_version = chromedriver_version.split(".")[0]
+                        if int(major_c_dr_version) < 96:
+                            message = (
+                                "You need to upgrade to a newer version of "
+                                "chromedriver to interact with Shadow root "
+                                "elements!\n(Current driver version is: %s)"
+                                "\n(Minimum driver version is: 96.*)"
+                                "\nTo upgrade: "
+                                '"seleniumbase install chromedriver latest"'
+                                % chromedriver_version
+                            )
+                            raise Exception(message)
+                    if timeout != 0.1:  # Skip wait for special 0.1 (See above)
+                        time.sleep(2)
+                    try:
+                        shadow_root = element.shadow_root
+                    except Exception:
+                        raise Exception(
+                            "Element {%s} has no shadow root!" % selector_chain
+                        )
+            else:  # This part won't work on Chrome 96 or newer.
+                # If using Chrome 96 or newer (and on an old Python version),
+                #     you'll need to upgrade in order to access Shadow roots.
+                # Firefox users will likely hit:
+                #     https://github.com/mozilla/geckodriver/issues/1711
+                #     When Firefox adds support, switch to element.shadow_root
+                shadow_root = self.execute_script(
+                    "return arguments[0].shadowRoot", element
+                )
             if timeout == 0.1 and not shadow_root:
                 raise Exception(
                     "Element {%s} has no shadow root!" % selector_chain
@@ -5755,12 +5801,20 @@ def __get_shadow_element(self, selector, timeout=None):
             selector_chain += "::shadow "
             selector_chain += selector_part
             try:
-                element = page_actions.wait_for_element_present(
-                    shadow_root,
-                    selector_part,
-                    by=By.CSS_SELECTOR,
-                    timeout=timeout,
-                )
+                if (
+                    selenium4
+                    and self.is_chromium()
+                    and int(self.__get_major_browser_version()) >= 96
+                ):
+                    element = shadow_root.find_element(
+                        By.CSS_SELECTOR, value=selector_part)
+                else:
+                    element = page_actions.wait_for_element_present(
+                        shadow_root,
+                        selector_part,
+                        by=By.CSS_SELECTOR,
+                        timeout=timeout,
+                    )
             except Exception:
                 msg = (
                     "Shadow DOM Element {%s} was not present after %s seconds!"
@@ -10101,6 +10155,15 @@ def __jquery_click(self, selector, by=By.CSS_SELECTOR):
         click_script = """jQuery('%s')[0].click();""" % selector
         self.safe_execute_script(click_script)
 
+    def __get_major_browser_version(self):
+        try:
+            version = self.driver.__dict__["caps"]["browserVersion"]
+        except Exception:
+            version = self.driver.__dict__["caps"]["version"]
+            self.driver.__dict__["caps"]["browserVersion"] = version
+        major_browser_version = version.split(".")[0]
+        return major_browser_version
+
     def __get_href_from_link_text(self, link_text, hard_fail=True):
         href = self.get_link_attribute(link_text, "href", hard_fail)
         if not href:
@@ -10276,6 +10339,7 @@ def __looks_like_a_page_url(self, url):
             or url.startswith("file:")
             or url.startswith("edge:")
             or url.startswith("opera:")
+            or url.startswith("view-source:")
         ):
             return True
         else:
diff --git a/setup.py b/setup.py

Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,2 @@`
`1`	`1`	`# seleniumbase package`
`2`		`-__version__ = "2.1.9"`
	`2`	`+__version__ = "2.2.0"`