Merge pull request #86 from MaineDSA/share_context

share context across entire run

Author: bmos

src/automation.py

@@ -33,10 +33,9 @@ async def create_browser_context() -> AsyncGenerator[BrowserContext, Any]:
         async with async_playwright() as p:
             context = await p.chromium.launch_persistent_context(
                 user_data_dir=temp_dir,
-                channel="chrome",  # CRITICAL: Use real Chrome, not Chromium
-                headless=False,  # CRITICAL: Don't use headless
-                no_viewport=True,  # CRITICAL: Use native resolution
-                # DO NOT add user_agent or extra_http_headers - Patchright handles this
+                channel="chrome",
+                headless=False,
+                no_viewport=True,
             )
 
             try:
@@ -46,17 +45,19 @@ async def create_browser_context() -> AsyncGenerator[BrowserContext, Any]:
 
 
 @asynccontextmanager
-async def get_browser_page() -> AsyncGenerator[Page, Any]:
+async def get_browser_page(context: BrowserContext, *, require_new_page: bool = False) -> AsyncGenerator[Page, Any]:
     """Create a browser page ready to use."""
-    async with create_browser_context() as context:
-        # Reuse existing page if available, otherwise create new one
-        pages = context.pages
-        if pages:
-            page = pages[0]
-        else:
-            page = await context.new_page()
+    # Reuse existing page if available, otherwise create new one
+    pages = context.pages
+    if pages and not require_new_page:
+        page = pages[0]
+    else:
+        page = await context.new_page()
 
+    try:
         yield page
+    finally:
+        await page.close()
 
 
 # Browser Automation - Scrolling and Navigation

src/main.py

@@ -3,7 +3,9 @@
 import asyncio
 import logging
 
-from src.automation import deduplicate_listings, get_browser_page, scrape_all_pages, simulate_human_behavior, sort_by_newest
+from patchright.async_api import BrowserContext
+
+from src.automation import create_browser_context, deduplicate_listings, get_browser_page, scrape_all_pages, simulate_human_behavior, sort_by_newest
 from src.config import Config, SubmissionType, load_configs
 from src.form_submission import submit_listings
 from src.scraper import PropertyListing
@@ -13,9 +15,9 @@
 logger = logging.getLogger(__name__)
 
 
-async def scrape_listings(config: Config) -> list[PropertyListing]:
+async def scrape_listings(context: BrowserContext, config: Config) -> list[PropertyListing]:
     """Scrape and deduplicate listings from Zillow."""
-    async with get_browser_page() as page:
+    async with get_browser_page(context) as page:
         logger.info("Loading search URL: %s...", config.search_url)
 
         await page.goto(config.search_url)
@@ -28,11 +30,11 @@ async def scrape_listings(config: Config) -> list[PropertyListing]:
         await sort_by_newest(page)
         all_listings = await scrape_all_pages(page)
 
-        logger.info("Deduplicating %s listings...", len(all_listings))
-        return deduplicate_listings(all_listings)
+    logger.info("Deduplicating %s listings...", len(all_listings))
+    return deduplicate_listings(all_listings)
 
 
-async def submit_listings_to_destination(config: Config, listings: list[PropertyListing]) -> None:
+async def submit_listings_to_destination(context: BrowserContext, config: Config, listings: list[PropertyListing]) -> None:
     """Submit listings based on configuration."""
     if not listings:
         logger.warning("No listings to submit")
@@ -48,26 +50,27 @@ async def submit_listings_to_destination(config: Config, listings: list[Property
         )
     elif config.submission_type == SubmissionType.FORM and isinstance(config.form_url, str):
         logger.info("Submitting %s listings to Google Form...", len(listings))
-        async with get_browser_page() as page:
+        async with get_browser_page(context) as page:
             await submit_listings(page, config.form_url, listings)
     else:
         logger.warning("No submission destination configured")
 
 
-async def scrape_and_submit(config: Config) -> None:
+async def scrape_and_submit(context: BrowserContext, config: Config) -> None:
     """Orchestrate scraping and submission workflow."""
-    listings = await scrape_listings(config)
-    await submit_listings_to_destination(config, listings)
+    listings = await scrape_listings(context, config)
+    await submit_listings_to_destination(context, config, listings)
 
 
-def main() -> None:
+async def main() -> None:
     """Load configurations and run scraper for each."""
     configs = load_configs()
-    for config in configs:
-        logger.info("Processing config: '%s'", config.config_name)
-        asyncio.run(scrape_and_submit(config))
-        logger.debug("Completed config: '%s'", config.config_name)
+    async with create_browser_context() as context:
+        for config in configs:
+            logger.info("Processing config: '%s'", config.config_name)
+            await scrape_and_submit(context, config)
+            logger.debug("Completed config: '%s'", config.config_name)
 
 
 if __name__ == "__main__":
-    main()
+    asyncio.run(main())

tests/test_main.py

@@ -1,16 +1,15 @@
 """Tests for main.py."""
 
 import logging
-from collections.abc import Coroutine
-from unittest.mock import patch
+from unittest.mock import AsyncMock, MagicMock, patch
 
 from _pytest.logging import LogCaptureFixture
 
 from src.config import Config, SubmissionType
 from src.main import main
 
 
-def test_main_with_mocked_configs(caplog: LogCaptureFixture) -> None:
+async def test_main_with_mocked_configs(caplog: LogCaptureFixture) -> None:
     """Test that main() works with multiple configs."""
     mock_config = [
         Config(
@@ -28,15 +27,28 @@ def test_main_with_mocked_configs(caplog: LogCaptureFixture) -> None:
         ),
     ]
 
-    def mock_run_impl(coro: Coroutine) -> None:
-        coro.close()  # Close the coroutine to prevent RuntimeWarning
+    with (
+        patch("src.main.load_configs", return_value=mock_config),
+        patch("src.main.scrape_and_submit", new_callable=AsyncMock) as mock_scrape_and_submit,
+        patch("src.main.create_browser_context") as mock_context,
+        caplog.at_level(logging.DEBUG),
+    ):
+        # Mock the async context manager
+        mock_browser_context = MagicMock()
+        mock_context.return_value.__aenter__.return_value = mock_browser_context
+        mock_context.return_value.__aexit__.return_value = None
 
-    with patch("src.main.load_configs", return_value=mock_config), patch("src.main.asyncio.run", side_effect=mock_run_impl) as mock_run:
-        with caplog.at_level(logging.DEBUG):
-            main()
+        # Run the actual main function
+        await main()
 
-        mock_run.assert_called()
-        assert mock_run.call_count == 2
+        # Verify scrape_and_submit was called twice (once per config)
+        assert mock_scrape_and_submit.call_count == 2
+
+        # Verify it was called with the correct configs
+        mock_scrape_and_submit.assert_any_call(mock_browser_context, mock_config[0])
+        mock_scrape_and_submit.assert_any_call(mock_browser_context, mock_config[1])
+
+        # Verify log messages
         assert "Processing config: 'config1.env'" in caplog.text
         assert "Completed config: 'config1.env'" in caplog.text
         assert "Processing config: 'config2.env'" in caplog.text