CDP Mode: Patch 91

examples/cdp_mode/ReadMe.md

@@ -202,9 +202,13 @@ with SB(uc=True, test=True, locale="en") as sb:
     sb.type('input[id="search-term"]', location)
     sb.sleep(1.2)
     sb.click('li[data-js="suggestion"]')
-    sb.sleep(1.2)
+    sb.sleep(0.6)
+    sb.click_if_visible('button[aria-label="Close"]')
+    sb.sleep(0.6)
     sb.click("button.be-button-shop")
-    sb.sleep(6)
+    sb.sleep(1)
+    sb.click_if_visible('[label="Find Hotels"]')
+    sb.sleep(5)
     card_info = 'div[data-booking-status="BOOKABLE"] [class*="HotelCard_info"]'
     hotels = sb.select_all(card_info)
     print("Hyatt Hotels in %s:" % location)

examples/cdp_mode/playwright/raw_gas_info_sync.py

@@ -0,0 +1,31 @@
+from playwright.sync_api import sync_playwright
+from seleniumbase import sb_cdp
+
+sb = sb_cdp.Chrome()
+endpoint_url = sb.get_endpoint_url()
+
+with sync_playwright() as p:
+    browser = p.chromium.connect_over_cdp(endpoint_url)
+    context = browser.contexts[0]
+    page = context.pages[0]
+    url = (
+        "https://www.gassaferegister.co.uk/gas-safety"
+        "/gas-safety-certificates-records/building-regulations-certificate"
+        "/order-replacement-building-regulations-certificate/"
+    )
+    page.goto(url)
+    sb.sleep(0.5)
+    sb.solve_captcha()
+    page.wait_for_selector("#SearchTerm")
+    allow_cookies = 'button:contains("Allow all cookies")'
+    sb.click_if_visible(allow_cookies, timeout=2)
+    sb.sleep(1)
+    page.fill("#SearchTerm", "Hydrogen")
+    page.click("button.search-button")
+    sb.sleep(3)
+    items = page.locator("div.search-result")
+    for i in range(items.count()):
+        item_text = items.nth(i).inner_text()
+        print(item_text.replace("\n\n", "\n") + "\n")
+    sb.scroll_to_bottom()
+    sb.sleep(1)

examples/cdp_mode/playwright/raw_reddit_sync.py

@@ -0,0 +1,25 @@
+from playwright.sync_api import sync_playwright
+from seleniumbase import sb_cdp
+
+sb = sb_cdp.Chrome(use_chromium=True)
+endpoint_url = sb.get_endpoint_url()
+
+with sync_playwright() as p:
+    browser = p.chromium.connect_over_cdp(endpoint_url)
+    context = browser.contexts[0]
+    page = context.pages[0]
+    search = "reddit+scraper"
+    url = f"https://www.reddit.com/r/webscraping/search/?q={search}"
+    page.goto(url)
+    sb.solve_captcha()  # Might not be needed
+    sb.sleep(1)
+    post_title = '[data-testid="post-title"]'
+    page.wait_for_selector(post_title)
+    for i in range(8):
+        sb.scroll_down(25)
+        sb.sleep(0.2)
+    print('*** Reddit Posts for "%s":' % search)
+    items = page.locator(post_title)
+    for i in range(items.count()):
+        item_text = items.nth(i).inner_text()
+        print("* " + item_text)

examples/cdp_mode/raw_easyjet.py

@@ -3,31 +3,29 @@
 with SB(uc=True, test=True, locale="en", ad_block=True) as sb:
     url = "https://www.easyjet.com/en/"
     sb.activate_cdp_mode(url)
-    sb.sleep(2)
-    sb.click_if_visible("button#ensCloseBanner")
-    sb.sleep(1.2)
+    sb.sleep(1.5)
+    sb.click_if_visible("button#ensRejectAds", timeout=2)
+    sb.sleep(1)
     sb.click('input[name="from"]')
-    sb.sleep(1.2)
+    sb.sleep(1)
     sb.type('input[name="from"]', "London Gatwick")
-    sb.sleep(0.6)
-    sb.click_if_visible("button#ensCloseBanner")
-    sb.sleep(0.6)
+    sb.sleep(1)
     sb.click('span[data-testid="airport-name"]')
-    sb.sleep(1.2)
+    sb.sleep(1)
     sb.type('input[name="to"]', "Paris")
-    sb.sleep(1.2)
+    sb.sleep(1)
     sb.click('span[data-testid="airport-name"]')
-    sb.sleep(1.2)
+    sb.sleep(1)
     sb.click('input[name="when"]')
-    sb.sleep(1.2)
+    sb.sleep(1)
     sb.click('[data-testid="month"]:last-of-type [aria-disabled="false"]')
-    sb.sleep(1.2)
+    sb.sleep(1)
     sb.click('[data-testid="month"]:last-of-type [aria-disabled="false"]')
-    sb.sleep(1.2)
+    sb.sleep(1)
     sb.click('button[data-testid="submit"]')
-    sb.sleep(4.2)
+    sb.sleep(4)
     sb.connect()
-    sb.sleep(1.2)
+    sb.sleep(1)
     for window in sb.driver.window_handles:
         sb.switch_to_window(window)
         if "/buy/flights" in sb.get_current_url():

examples/cdp_mode/raw_gas_records.py

@@ -8,21 +8,18 @@
         "/order-replacement-building-regulations-certificate/"
     )
     sb.activate_cdp_mode(url)
-    sb.sleep(0.6)
+    sb.sleep(0.5)
     sb.solve_captcha()
-    sb.sleep(1)
     sb.wait_for_element("#SearchTerm", timeout=5)
     sb.sleep(2)
     allow_cookies = 'button:contains("Allow all cookies")'
     sb.click_if_visible(allow_cookies, timeout=2)
-    sb.sleep(1.2)
+    sb.sleep(1)
     sb.press_keys("#SearchTerm", "Hydrogen")
-    sb.sleep(0.5)
     sb.click("button.search-button")
     sb.sleep(3)
     results = sb.find_elements("div.search-result")
     for result in results:
-        print(result.text.replace(" " * 12, " ").strip())
-        print()
+        print(result.text.replace(" " * 12, " ").strip() + "\n")
     sb.scroll_to_bottom()
     sb.sleep(1)

examples/cdp_mode/raw_hyatt.py

@@ -12,9 +12,13 @@
     sb.type('input[id="search-term"]', location)
     sb.sleep(1.2)
     sb.click('li[data-js="suggestion"]')
-    sb.sleep(1.2)
+    sb.sleep(0.6)
+    sb.click_if_visible('button[aria-label="Close"]')
+    sb.sleep(0.6)
     sb.click("button.be-button-shop")
-    sb.sleep(6)
+    sb.sleep(1)
+    sb.click_if_visible('[label="Find Hotels"]')
+    sb.sleep(5)
     card_info = 'div[data-booking-status="BOOKABLE"] [class*="HotelCard_info"]'
     hotels = sb.select_all(card_info)
     print("Hyatt Hotels in %s:" % location)

examples/cdp_mode/raw_priceline.py

@@ -1,6 +1,6 @@
 from seleniumbase import SB
 
-with SB(uc=True, test=True, locale="en") as sb:
+with SB(uc=True, test=True, locale="en", guest=True, ad_block=True) as sb:
     url = "https://www.priceline.com"
     sb.activate_cdp_mode(url)
     sb.sleep(1.8)
@@ -9,19 +9,15 @@
     location = "Portland, OR"
     selection = "Oregon, United States"  # (Dropdown option)
     sb.press_keys('input[name="endLocation"]', location)
-    sb.sleep(1.5)
+    sb.sleep(0.5)
     sb.click_if_visible('input[name="endLocation"]')
-    sb.sleep(0.6)
-    sb.click(selection)
-    sb.sleep(1.5)
-    sb.click('button[aria-label="Dismiss calendar"]')
     sb.sleep(0.5)
+    sb.click(selection)
+    sb.scroll_down(25)
+    sb.click_if_visible('button[aria-label="Dismiss calendar"]')
+    sb.click_if_visible("div.sidebar-iframe-close")
+    sb.click_if_visible('div[aria-label="Close Modal"]')
     sb.click('button[data-testid="HOTELS_SUBMIT_BUTTON"]')
-    sb.sleep(0.5)
-    if sb.is_element_visible('[aria-label="Close Modal"]'):
-        sb.click('[aria-label="Close Modal"]')
-        sb.sleep(0.5)
-        sb.click('button[data-testid="HOTELS_SUBMIT_BUTTON"]')
     sb.sleep(4.8)
     if len(sb.cdp.get_tabs()) > 1:
         sb.cdp.close_active_tab()

examples/cdp_mode/raw_ralphlauren.py

@@ -2,9 +2,8 @@
 
 with SB(uc=True, test=True, locale="en") as sb:
     url = "https://www.ralphlauren.com.au/"
-    sb.activate_cdp_mode()
-    sb.open(url)
-    sb.sleep(1.2)
+    sb.activate_cdp_mode(url)
+    sb.sleep(1.6)
     if not sb.is_element_present('[title="Locate Stores"]'):
         sb.evaluate("window.location.reload();")
         sb.sleep(1.2)
@@ -15,8 +14,8 @@
     sb.click('a:contains("%s")' % search)
     sb.sleep(3.8)
     for i in range(6):
-        sb.scroll_down(34)
-        sb.sleep(0.25)
+        sb.scroll_down(25)
+        sb.sleep(0.2)
     print('*** Ralph Lauren Search for "%s":' % search)
     unique_item_text = []
     items = sb.find_elements('div.product-data')

examples/cdp_mode/raw_zoro.py

@@ -16,14 +16,15 @@
     sb.press_keys(search_box, search)
     sb.sleep(0.6)
     sb.click('button[data-za="searchButton"]')
-    sb.sleep(3.8)
+    sb.sleep(3.2)
+    sb.wait_for_element('[data-za="product-cards-list"]', timeout=5)
     print('*** Zoro Search for "%s":' % search)
     print('    (Results must contain "%s".)' % required_text)
     unique_item_text = []
-    items = sb.find_elements('div[data-za="search-product-card"]')
+    items = sb.find_elements('[data-za="search-product-card"]')
     for item in items:
         if required_text in item.text:
-            description = item.querySelector('div[data-za="product-title"]')
+            description = item.querySelector('[data-za="product-title"]')
             if description and description.text not in unique_item_text:
                 unique_item_text.append(description.text)
                 print("* " + description.text)

examples/presenter/uc_presentation_4.py

@@ -768,7 +768,9 @@ def test_presentation_4(self):
         )
         self.begin_presentation(filename="uc_presentation.html")
 
-        with SB(uc=True, test=True, locale="en") as sb:
+        with SB(
+            uc=True, test=True, locale="en", guest=True, ad_block=True
+        ) as sb:
             url = "https://www.priceline.com"
             sb.activate_cdp_mode(url)
             sb.sleep(1.8)
@@ -777,19 +779,15 @@ def test_presentation_4(self):
             location = "Portland, Oregon, US"
             selection = "Oregon, United States"  # (Dropdown option)
             sb.press_keys('input[name="endLocation"]', location)
-            sb.sleep(1.5)
+            sb.sleep(0.5)
             sb.click_if_visible('input[name="endLocation"]')
-            sb.sleep(0.6)
-            sb.click(selection)
-            sb.sleep(1.5)
-            sb.click('button[aria-label="Dismiss calendar"]')
             sb.sleep(0.5)
+            sb.click(selection)
+            sb.scroll_down(25)
+            sb.click_if_visible('button[aria-label="Dismiss calendar"]')
+            sb.click_if_visible("div.sidebar-iframe-close")
+            sb.click_if_visible('div[aria-label="Close Modal"]')
             sb.click('button[data-testid="HOTELS_SUBMIT_BUTTON"]')
-            sb.sleep(0.5)
-            if sb.is_element_visible('[aria-label="Close Modal"]'):
-                sb.click('[aria-label="Close Modal"]')
-                sb.sleep(0.5)
-                sb.click('button[data-testid="HOTELS_SUBMIT_BUTTON"]')
             sb.sleep(4.8)
             if len(sb.cdp.get_tabs()) > 1:
                 sb.cdp.close_active_tab()

seleniumbase/__version__.py

@@ -1,2 +1,2 @@
 # seleniumbase package
-__version__ = "4.45.11"
+__version__ = "4.45.12"

seleniumbase/core/sb_cdp.py

@@ -795,15 +795,13 @@ def click_if_visible(self, selector, timeout=0):
         if self.is_element_visible(selector):
             with suppress(Exception):
                 self.click(selector, timeout=1)
+        elif timeout == 0:
+            return
         else:
             with suppress(Exception):
-                element = self.find_element(selector, timeout=timeout)
-                self.sleep(0.1)
-                element.scroll_into_view()
-                self.sleep(0.1)
-                element.click()
-                self.__slow_mode_pause_if_set()
-                self.loop.run_until_complete(self.page.wait())
+                self.find_element(selector, timeout=timeout)
+                if self.is_element_visible(selector):
+                    self.click(selector, timeout=1)
 
     def click_visible_elements(self, selector, limit=0):
         """Finds all matching page elements and clicks visible ones in order.
@@ -2068,14 +2066,35 @@ def __cdp_click_incapsula_hcaptcha(self):
         time.sleep(0.05)
         x_offset = 30
         y_offset = 36
+        was_clicked = False
         gui_lock = FileLock(constants.MultiBrowser.PYAUTOGUILOCK)
         with gui_lock:  # Prevent issues with multiple processes
             self.bring_active_window_to_front()
-            time.sleep(0.05)
+            time.sleep(0.056)
+            if "--debug" in sys.argv:
+                displayed_selector = "`%s`" % selector
+                if '"' not in selector:
+                    displayed_selector = '"%s"' % selector
+                elif "'" not in selector:
+                    displayed_selector = "'%s'" % selector
+                print(
+                    " <DEBUG> click_with_offset(%s, %s, %s)"
+                    % (displayed_selector, x_offset, y_offset)
+                )
             with suppress(Exception):
                 element.click_with_offset(x_offset, y_offset)
-                time.sleep(0.2)
-                return True
+                was_clicked = True
+                time.sleep(0.056)
+        if was_clicked:
+            # Wait a moment for the click to succeed
+            time.sleep(0.25)
+            self.__slow_mode_pause_if_set()
+            self.loop.run_until_complete(self.page.wait())
+            if "--debug" in sys.argv:
+                print(" <DEBUG> hCaptcha was clicked!")
+            return True
+        if "--debug" in sys.argv:
+            print(" <DEBUG> hCaptcha was NOT clicked!")
         return False
 
     def solve_captcha(self):

seleniumbase/fixtures/base_case.py

@@ -92,6 +92,7 @@ def test_anything(self):
 
 logging.getLogger("requests").setLevel(logging.ERROR)
 logging.getLogger("urllib3").setLevel(logging.ERROR)
+logging.getLogger("websocket").setLevel(logging.CRITICAL)
 urllib3.disable_warnings()
 LOGGER.setLevel(logging.WARNING)
 is_linux = shared_utils.is_linux()