feat(document_loaders): add flexible timeout to PlaywrightURLLoader (#104)

## Description
This PR enhances the `PlaywrightURLLoader` by adding configurable
timeout and page load strategy options, making it more flexible for
handling dynamic web pages. This addresses issue #103.

### Changes
- Added `timeout` parameter (default: 30000ms) to control page
navigation timeout
- Added `wait_until` parameter to control when navigation is considered
complete
- Supported `wait_until` options:
  - `"load"` (default): wait for the "load" event
  - `"domcontentloaded"`: wait for the "DOMContentLoaded" event
- `"networkidle"`: wait until there are no network connections for at
least 500ms
  - `"commit"`: wait for the first network request to be sent

### Why
The current implementation has a hardcoded 30-second timeout, which can
be insufficient for heavy dynamic pages. This change allows users to:
- Set longer timeouts for complex pages
- Choose appropriate page load strategies based on their needs
- Better handle dynamic content loading

### Real-World Examples
This PR solves timeout issues with various types of websites:

1. Weather websites:
```python
loader = PlaywrightURLLoader(
    urls=["https://weather.com/en-IN/weather/tenday/l/Chennai+Tamil+Nadu?canonicalCityId=251b7b4afedf19f747b425e048038eb1"],
    timeout=60000,  # 60 second timeout
    wait_until="domcontentloaded"
)
```

2. Dynamic news sites:
```python
loader = PlaywrightURLLoader(
    urls=["https://www.reuters.com/markets/"],
    timeout=45000,
    wait_until="networkidle"
)
```

3. E-commerce sites:
```python
loader = PlaywrightURLLoader(
    urls=["https://www.amazon.com/dp/B08N5KWB9H"],
    timeout=90000,  # 90 second timeout for complex product pages
    wait_until="load"
)
```

### Testing
- Added new test cases for both sync and async methods
- Maintained backward compatibility
- All existing tests pass
- Tested with various real-world websites

### Related Issues
Closes #103

---------

Co-authored-by: Parth Pathak <[email protected]>
Co-authored-by: Mason Daugherty <[email protected]>
Co-authored-by: Mason Daugherty <[email protected]>

Author: 3 people

libs/community/langchain_community/document_loaders/url_playwright.py

@@ -116,6 +116,16 @@ class PlaywrightURLLoader(BaseLoader):
             through the specified proxy.
         browser_session (Optional[Union[str, os.PathLike[str]]]): Path to a file with
             browser session data that can be used to restore the browser session.
+        timeout (Optional[int]): Timeout in milliseconds for page navigation.
+        wait_until (Optional[str]): When to consider navigation succeeded.
+
+            Can be one of:
+
+            - "load": wait for the "load" event to fire
+            - "domcontentloaded": wait for the "DOMContentLoaded" event to fire
+            - "networkidle": wait until there are no network connections for at least
+                500ms
+            - "commit": wait for the first network request to be sent
 
     Example:
         .. code-block:: python
@@ -128,7 +138,12 @@ class PlaywrightURLLoader(BaseLoader):
                 "username": "username",
                 "password": "password"
             }
-            loader = PlaywrightURLLoader(urls, proxy=proxy)
+            loader = PlaywrightURLLoader(
+                urls=urls,
+                proxy=proxy,
+                timeout=60000,  # 60 second timeout
+                wait_until="domcontentloaded"  # Wait for DOM content to load
+            )
             data = loader.load()
     """
 
@@ -141,6 +156,8 @@ def __init__(
         evaluator: Optional[PlaywrightEvaluator] = None,
         proxy: Optional[Dict[str, str]] = None,
         browser_session: Optional[Union[str, os.PathLike[str]]] = None,
+        timeout: Optional[int] = 30000,
+        wait_until: Optional[str] = "load",
     ):
         """Load a list of URLs using Playwright."""
         try:
@@ -156,6 +173,8 @@ def __init__(
         self.headless = headless
         self.proxy = proxy
         self.browser_session = browser_session
+        self.timeout = timeout
+        self.wait_until = wait_until
 
         if remove_selectors and evaluator:
             raise ValueError(
@@ -189,11 +208,13 @@ def lazy_load(self) -> Iterator[Document]:
             for url in self.urls:
                 try:
                     page = context.new_page()
-                    response = page.goto(url)
+                    response = page.goto(
+                        url, timeout=self.timeout, wait_until=self.wait_until
+                    )
                     if response is None:
                         raise ValueError(f"page.goto() returned None for url {url}")
 
-                    page.wait_for_load_state("load")
+                    page.wait_for_load_state(self.wait_until, timeout=self.timeout)
 
                     text = self.evaluator.evaluate(page, browser, response)
                     page.close()
@@ -244,11 +265,15 @@ async def alazy_load(self) -> AsyncIterator[Document]:
             for url in self.urls:
                 try:
                     page = await context.new_page()
-                    response = await page.goto(url)
+                    response = await page.goto(
+                        url, timeout=self.timeout, wait_until=self.wait_until
+                    )
                     if response is None:
                         raise ValueError(f"page.goto() returned None for url {url}")
 
-                    await page.wait_for_load_state("load")
+                    await page.wait_for_load_state(
+                        self.wait_until, timeout=self.timeout
+                    )
 
                     text = await self.evaluator.evaluate_async(page, browser, response)
                     await page.close()

libs/community/tests/integration_tests/document_loaders/test_url_playwright.py

@@ -42,6 +42,20 @@ def test_playwright_url_loader() -> None:
     assert len(docs) > 0
 
 
+def test_playwright_url_loader_with_timeout() -> None:
+    """Test Playwright URL loader with custom timeout and wait_until."""
+    urls = ["https://techmeme.com"]
+    loader = PlaywrightURLLoader(
+        urls=urls,
+        timeout=60000,  # 60 second timeout
+        wait_until="domcontentloaded",  # Wait for DOM content to load
+        continue_on_failure=False,
+        headless=True,
+    )
+    docs = loader.load()
+    assert len(docs) > 0
+
+
 async def test_playwright_async_url_loader() -> None:
     """Test Playwright async URL loader."""
     urls = [
@@ -60,6 +74,20 @@ async def test_playwright_async_url_loader() -> None:
     assert len(docs) > 0
 
 
+async def test_playwright_async_url_loader_with_timeout() -> None:
+    """Test Playwright async URL loader with custom timeout and wait_until."""
+    urls = ["https://techmeme.com"]
+    loader = PlaywrightURLLoader(
+        urls=urls,
+        timeout=60000,  # 60 second timeout
+        wait_until="domcontentloaded",  # Wait for DOM content to load
+        continue_on_failure=False,
+        headless=True,
+    )
+    docs = await loader.aload()
+    assert len(docs) > 0
+
+
 def test_playwright_url_loader_with_custom_evaluator() -> None:
     """Test Playwright URL loader with a custom evaluator."""
     urls = ["https://www.youtube.com/watch?v=dQw4w9WgXcQ"]