Merge pull request #98 from MaineDSA/handle_modal

close modal popup if present

Author: bmos

src/automation.py

@@ -19,6 +19,7 @@
     MIN_SCROLL_UP,
     MIN_WAIT_TIME,
     PROBABILITY_SCROLL_UP,
+    ZillowParseError,
 )
 from src.scraper import PropertyListing, ZillowHomeFinder
 
@@ -64,6 +65,20 @@ async def get_browser_page(context: BrowserContext, *, require_new_page: bool =
 # Browser Automation - Scrolling and Navigation
 
 
+async def close_modal_if_present(page: Page) -> None:
+    """Close modal dialog by clicking button with class containing 'CloseButton', if present."""
+    try:
+        close_button = page.locator("button[class*='CloseButton']").first
+        is_visible = await close_button.is_visible()
+        if not is_visible:
+            msg = "Popup modal blocked page loading, cannot scrape."
+            raise ZillowParseError(msg)
+        logger.debug("Popup modal detected, closing it")
+        await close_button.click()
+    except TimeoutError as e:
+        logger.debug("No CloseButton modal found or could not close: %s", e)
+
+
 async def get_property_card_count(page: Page) -> int:
     """Get the current count of loaded property cards."""
     cards = await page.query_selector_all('article[data-test="property-card"]')

src/main.py

@@ -5,7 +5,15 @@
 
 from patchright.async_api import BrowserContext
 
-from src.automation import create_browser_context, deduplicate_listings, get_browser_page, scrape_all_pages, simulate_human_behavior, sort_by_newest
+from src.automation import (
+    close_modal_if_present,
+    create_browser_context,
+    deduplicate_listings,
+    get_browser_page,
+    scrape_all_pages,
+    simulate_human_behavior,
+    sort_by_newest,
+)
 from src.config import Config, SubmissionType, load_configs
 from src.form_submission import submit_listings
 from src.scraper import PropertyListing
@@ -26,6 +34,8 @@ async def scrape_listings(context: BrowserContext, config: Config) -> list[Prope
             error_msg = "CAPTCHA detected, cannot continue."
             raise BaseException(error_msg)
 
+        await close_modal_if_present(page)
+
         logger.info("Scraping all listings...")
         await sort_by_newest(page)
         all_listings = await scrape_all_pages(page)