Merge pull request #85 from MaineDSA/human_simulation

replace waiting with mouse movements and waiting

Author: bmos

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "zillow_scraper"
-version = "2.1.0"
+version = "2.2.0"
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.11"
 license = { file = "LICENSE" }

src/automation.py

@@ -99,17 +99,34 @@ async def scroll_down(page: Page, amount: int) -> None:
     )
 
 
+async def simulate_human_behavior(page: Page) -> None:
+    """Simulate human-like mouse movements and pauses."""
+    window_dimensions = await page.evaluate("""
+        () => ({
+            width: window.innerWidth,
+            height: window.innerHeight
+        })
+    """)
+
+    # Move mouse to random positions within the window
+    x = cryptogen.randint(100, window_dimensions["width"] - 100)
+    y = cryptogen.randint(100, window_dimensions["height"] - 100)
+    await page.mouse.move(x, y)
+
+    await page.wait_for_timeout(cryptogen.randint(MIN_WAIT_TIME, MAX_WAIT_TIME))
+
+
 async def perform_human_like_scroll(page: Page) -> None:
     """Perform a human-like scrolling action with random variations."""
     scroll_amount = cryptogen.randint(MIN_SCROLL_DOWN, MAX_SCROLL_DOWN)
     await scroll_down(page, scroll_amount)
-    await page.wait_for_timeout(cryptogen.randint(MIN_WAIT_TIME, MAX_WAIT_TIME))
+    await simulate_human_behavior(page)
 
     # Occasionally scroll back up
     if cryptogen.random() < PROBABILITY_SCROLL_UP:
         back_scroll = cryptogen.randint(MIN_SCROLL_UP, MAX_SCROLL_UP)
         await scroll_down(page, -back_scroll)
-        await page.wait_for_timeout(cryptogen.randint(MIN_WAIT_TIME, MAX_WAIT_TIME))
+        await simulate_human_behavior(page)
 
 
 async def scroll_to_top(page: Page) -> None:
@@ -122,7 +139,7 @@ async def scroll_to_top(page: Page) -> None:
             window.scrollTo(0, 0);
         }
     """)
-    await page.wait_for_timeout(cryptogen.randint(MIN_WAIT_TIME, MAX_WAIT_TIME))
+    await simulate_human_behavior(page)
 
 
 async def scroll_and_load_listings(page: Page, max_entries: int = 100, max_no_change: int = 3, max_scroll_attempts: int = 50) -> None:

src/main.py

@@ -2,17 +2,15 @@
 
 import asyncio
 import logging
-from random import SystemRandom
 
-from src.automation import deduplicate_listings, get_browser_page, scrape_all_pages, sort_by_newest
+from src.automation import deduplicate_listings, get_browser_page, scrape_all_pages, simulate_human_behavior, sort_by_newest
 from src.config import Config, SubmissionType, load_configs
 from src.form_submission import submit_listings
 from src.scraper import PropertyListing
 from src.sheets_submission import SheetsSubmitter
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s:zillow_scraper:%(name)s:%(message)s")
 logger = logging.getLogger(__name__)
-cryptogen = SystemRandom()
 
 
 async def scrape_listings(config: Config) -> list[PropertyListing]:
@@ -21,7 +19,7 @@ async def scrape_listings(config: Config) -> list[PropertyListing]:
         logger.info("Loading search URL: %s...", config.search_url)
 
         await page.goto(config.search_url)
-        await page.wait_for_timeout(cryptogen.randint(1000, 3500))
+        await simulate_human_behavior(page)
         if await page.get_by_text("Press & Hold").count() > 0:
             error_msg = "CAPTCHA detected, cannot continue."
             raise BaseException(error_msg)