consolidate timeout pauses to simulate_human_behavior

Author: bmos

src/automation.py

@@ -193,6 +193,7 @@ async def scroll_and_load_listings(page: Page, max_entries: int = 100, max_no_ch
     logger.debug("Lazy loading complete. Total property cards loaded: %s", final_count)
 
     await scroll_to_top(page)
+    await simulate_human_behavior(page)
 
 
 async def check_and_click_next_page(page: Page) -> bool:
@@ -246,7 +247,7 @@ async def sort_by_newest(page: Page) -> None:
 
     await sort_button.click()
     await page.wait_for_load_state()
-    await page.wait_for_timeout(1.23)
+    await simulate_human_behavior(page)
 
     newest_button = page.get_by_text("Newest")
     if not newest_button:

src/form_submission.py

@@ -7,7 +7,8 @@
 from patchright.async_api import TimeoutError as PlaywrightTimeoutError
 from tqdm import tqdm
 
-from src.constants import MAX_WAIT_TIME, MIN_WAIT_TIME, GoogleFormConstants
+from src.automation import simulate_human_behavior
+from src.constants import GoogleFormConstants
 from src.scraper import PropertyListing
 
 logger = logging.getLogger(__name__)
@@ -17,7 +18,7 @@
 async def _submit_single_listing(page: Page, url: str, listing: PropertyListing) -> None:
     """Submit a single listing to the Google Form."""
     await page.goto(url)
-    await page.wait_for_timeout(cryptogen.randint(MIN_WAIT_TIME, MAX_WAIT_TIME))
+    await simulate_human_behavior(page)
 
     await page.fill(GoogleFormConstants.ADDRESS_INPUT_XPATH, listing.address)
     await page.fill(GoogleFormConstants.PRICE_INPUT_XPATH, listing.price)
@@ -31,7 +32,7 @@ async def _submit_single_listing(page: Page, url: str, listing: PropertyListing)
         error_msg = f"Form submission confirmation not received for {listing.address}"
         raise PlaywrightTimeoutError(error_msg) from e
 
-    await page.wait_for_timeout(cryptogen.randint(MIN_WAIT_TIME, MAX_WAIT_TIME))
+    await simulate_human_behavior(page)
 
 
 async def submit_listings(page: Page, form_url: str, listings: list[PropertyListing]) -> None:

src/main.py

@@ -39,7 +39,6 @@ async def scrape_listings(context: BrowserContext, config: Config) -> list[Prope
 
         logger.info("Scraping all listings...")
         await sort_by_newest(page)
-        await page.wait_for_timeout(0.84)
         all_listings = await scrape_all_pages(page)
 
     logger.info("Deduplicating %s listings...", len(all_listings))

tests/test_form_submission.py

@@ -53,7 +53,7 @@ def wait_for_selector_side_effect(*_, **__) -> AsyncMock:
     mock_page.wait_for_selector.side_effect = wait_for_selector_side_effect
     form_url = "https://example.com/form"
 
-    with caplog.at_level("INFO"), patch("src.form_submission.cryptogen.randint", return_value=250):
+    with caplog.at_level("INFO"), patch("src.automation.cryptogen.randint", return_value=250):
         await submit_listings(mock_page, form_url, sample_listings)
 
     assert mock_page.goto.call_count == 3
@@ -67,7 +67,7 @@ async def test_submit_listings_all_succeed(caplog: LogCaptureFixture, sample_lis
     mock_page = AsyncMock()
     form_url = "https://example.com/form"
 
-    with caplog.at_level("INFO"), patch("src.form_submission.cryptogen.randint", return_value=250):
+    with caplog.at_level("INFO"), patch("src.automation.cryptogen.randint", return_value=250):
         await submit_listings(mock_page, form_url, sample_listings)
 
     assert "3 successful, 0 failed" in caplog.text
@@ -82,7 +82,7 @@ async def test_submit_listings_all_fail(caplog: LogCaptureFixture, sample_listin
     mock_page.wait_for_selector.side_effect = PlaywrightTimeoutError("Timeout")
     form_url = "https://example.com/form"
 
-    with caplog.at_level("INFO"), patch("src.form_submission.cryptogen.randint", return_value=250):
+    with caplog.at_level("INFO"), patch("src.automation.cryptogen.randint", return_value=250):
         await submit_listings(mock_page, form_url, sample_listings)
 
     assert "0 successful, 3 failed" in caplog.text
@@ -107,7 +107,7 @@ async def test_submit_single_listing_flow_order() -> None:
     mock_page.wait_for_selector.side_effect = lambda *_, **__: call_order.append("wait_for_selector")
     mock_page.wait_for_timeout.side_effect = lambda _: call_order.append("wait_for_timeout")
 
-    with patch("src.form_submission.cryptogen.randint", return_value=250):
+    with patch("src.automation.cryptogen.randint", return_value=250):
         await _submit_single_listing(mock_page, form_url, listing)
 
     # Verify the sequence
@@ -123,30 +123,6 @@ async def test_submit_single_listing_flow_order() -> None:
     ]
 
 
-@pytest.mark.asyncio
-async def test_submit_listings_uses_random_waits() -> None:
-    """Test that random wait times are used between submissions."""
-    mock_page = AsyncMock()
-    listings = [
-        PropertyListing("Addr1", "$1000", "1000", "http://link1"),
-        PropertyListing("Addr2", "$2000", "2000", "http://link2"),
-    ]
-    form_url = "https://example.com/form"
-
-    wait_times: list[int] = []
-    mock_page.wait_for_timeout.side_effect = wait_times.append
-
-    with patch("src.form_submission.cryptogen.randint") as mock_randint:
-        # Return different values for each call
-        mock_randint.side_effect = [100, 150, 200, 250]
-        await submit_listings(mock_page, form_url, listings)
-
-    # Should have wait_for_timeout called multiple times (2 per listing)
-    assert len(wait_times) == 4
-    # Verify the random values were used
-    assert wait_times == [100, 150, 200, 250]
-
-
 @pytest.mark.parametrize(
     "empty_list_arg",
     [
@@ -177,7 +153,7 @@ async def test_submit_single_listing_field_mapping(mock_page: AsyncMock) -> None
     listing = PropertyListing(address="742 Evergreen Terrace", price="$2,500/mo", median_price="2500", link="https://zillow.com/listing/999")
     form_url = "https://example.com/form"
 
-    with patch("src.form_submission.cryptogen.randint", return_value=250):
+    with patch("src.automation.cryptogen.randint", return_value=250):
         await _submit_single_listing(mock_page, form_url, listing)
 
     fill_calls = mock_page.fill.call_args_list