docs: Add guide about integrating Stagehand (#1290)

### Description

- Add guide about integrating `stagehand-python` v.0.4.0

### Issues

- Closes: #1278

Author: Mantisus

docs/guides/code_examples/playwright_crawler_stagehand/__init__.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING, Any, cast
+
+from stagehand.context import StagehandContext
+from typing_extensions import override
+
+from crawlee.browsers import (
+    PlaywrightBrowserController,
+    PlaywrightBrowserPlugin,
+    PlaywrightPersistentBrowser,
+)
+
+from .support_classes import CrawleeStagehandPage
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+    from playwright.async_api import Page
+    from stagehand import Stagehand
+
+    from crawlee.proxy_configuration import ProxyInfo
+
+
+class StagehandBrowserController(PlaywrightBrowserController):
+    @override
+    def __init__(
+        self, browser: PlaywrightPersistentBrowser, stagehand: Stagehand, **kwargs: Any
+    ) -> None:
+        # Initialize with browser context instead of browser instance
+        super().__init__(browser, **kwargs)
+
+        self._stagehand = stagehand
+        self._stagehand_context: StagehandContext | None = None
+
+    @override
+    async def new_page(
+        self,
+        browser_new_context_options: Mapping[str, Any] | None = None,
+        proxy_info: ProxyInfo | None = None,
+    ) -> Page:
+        # Initialize browser context if not already done
+        if not self._browser_context:
+            self._browser_context = await self._create_browser_context(
+                browser_new_context_options=browser_new_context_options,
+                proxy_info=proxy_info,
+            )
+
+        # Initialize Stagehand context if not already done
+        if not self._stagehand_context:
+            self._stagehand_context = await StagehandContext.init(
+                self._browser_context, self._stagehand
+            )
+
+        # Create a new page using Stagehand context
+        page = await self._stagehand_context.new_page()
+
+        pw_page = page._page  # noqa: SLF001
+
+        # Handle page close event
+        pw_page.on(event='close', f=self._on_page_close)
+
+        # Update internal state
+        self._pages.append(pw_page)
+        self._last_page_opened_at = datetime.now(timezone.utc)
+
+        self._total_opened_pages += 1
+
+        # Wrap StagehandPage to provide Playwright Page interface
+        return cast('Page', CrawleeStagehandPage(page))
+
+
+class StagehandPlugin(PlaywrightBrowserPlugin):
+    """Browser plugin that integrates Stagehand with Crawlee's browser management."""
+
+    @override
+    def __init__(self, stagehand: Stagehand, **kwargs: Any) -> None:
+        super().__init__(**kwargs)
+
+        self._stagehand = stagehand
+
+    @override
+    async def new_browser(self) -> StagehandBrowserController:
+        if not self._playwright:
+            raise RuntimeError('Playwright browser plugin is not initialized.')
+
+        browser = PlaywrightPersistentBrowser(
+            # Stagehand can run only on a Chromium-based browser.
+            self._playwright.chromium,
+            self._user_data_dir,
+            self._browser_launch_options,
+        )
+
+        # Return custom controller with Stagehand
+        return StagehandBrowserController(
+            browser=browser,
+            stagehand=self._stagehand,
+            header_generator=None,
+            fingerprint_generator=None,
+        )

docs/guides/code_examples/playwright_crawler_stagehand/browser_classes.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+import asyncio
+import os
+from typing import cast
+
+from stagehand import StagehandConfig, StagehandPage
+
+from crawlee import ConcurrencySettings
+from crawlee.browsers import BrowserPool
+from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
+
+from .browser_classes import StagehandPlugin
+from .support_classes import CrawleeStagehand
+
+
+async def main() -> None:
+    # Configure local Stagehand with Gemini model
+    config = StagehandConfig(
+        env='LOCAL',
+        model_name='google/gemini-2.5-flash-preview-05-20',
+        model_api_key=os.getenv('GEMINI_API_KEY'),
+    )
+
+    # Create Stagehand instance
+    stagehand = CrawleeStagehand(config)
+
+    # Create crawler with custom browser pool using Stagehand
+    crawler = PlaywrightCrawler(
+        # Limit the crawl to max requests. Remove or increase it for crawling all links.
+        max_requests_per_crawl=10,
+        # Custom browser pool. Gives users full control over browsers used by the crawler.
+        concurrency_settings=ConcurrencySettings(max_tasks_per_minute=10),
+        browser_pool=BrowserPool(
+            plugins=[
+                StagehandPlugin(stagehand, browser_launch_options={'headless': True})
+            ],
+        ),
+    )
+
+    # Define the default request handler, which will be called for every request.
+    @crawler.router.default_handler
+    async def request_handler(context: PlaywrightCrawlingContext) -> None:
+        context.log.info(f'Processing {context.request.url} ...')
+
+        # Cast to StagehandPage for proper type hints in IDE
+        page = cast('StagehandPage', context.page)
+
+        # Use regular Playwright method
+        playwright_title = await page.title()
+        context.log.info(f'Playwright page title: {playwright_title}')
+
+        # highlight-start
+        # Use AI-powered extraction with natural language
+        gemini_title = await page.extract('Extract page title')
+        context.log.info(f'Gemini page title: {gemini_title}')
+        # highlight-end
+
+        await context.enqueue_links()
+
+    # Run the crawler with the initial list of URLs.
+    await crawler.run(['https://crawlee.dev/'])
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

docs/guides/code_examples/playwright_crawler_stagehand/stagehand_run.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from stagehand import Stagehand, StagehandPage
+
+if TYPE_CHECKING:
+    from types import TracebackType
+
+
+class CrawleeStagehandPage:
+    """StagehandPage wrapper for Crawlee."""
+
+    def __init__(self, page: StagehandPage) -> None:
+        self._page = page
+
+    async def goto(
+        self,
+        url: str,
+        *,
+        referer: str | None = None,
+        timeout: int | None = None,
+        wait_until: str | None = None,
+    ) -> Any:
+        """Navigate to the specified URL."""
+        # Override goto to return navigation result that `PlaywrightCrawler` expects
+        return await self._page._page.goto(  # noqa: SLF001
+            url,
+            referer=referer,
+            timeout=timeout,
+            wait_until=wait_until,
+        )
+
+    def __getattr__(self, name: str) -> Any:
+        """Delegate all other methods to the underlying StagehandPage."""
+        return getattr(self._page, name)
+
+    async def __aenter__(self) -> CrawleeStagehandPage:
+        """Enter the context manager."""
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        exc_traceback: TracebackType | None,
+    ) -> None:
+        await self._page.close()
+
+
+class CrawleeStagehand(Stagehand):
+    """Stagehand wrapper for Crawlee to disable the launch of Playwright."""
+
+    async def init(self) -> None:
+        # Skip Stagehand's own Playwright initialization
+        # Let Crawlee's PlaywrightBrowserPlugin manage the browser lifecycle
+        self._initialized = True

docs/guides/code_examples/playwright_crawler_stagehand/support_classes.py

@@ -0,0 +1,66 @@
+---
+id: playwright-crawler-stagehand
+title: Playwright with Stagehand
+description: How to integrate Stagehand AI-powered automation with PlaywrightCrawler.
+---
+
+import ApiLink from '@site/src/components/ApiLink';
+import CodeBlock from '@theme/CodeBlock';
+
+import SupportClasses from '!!raw-loader!./code_examples/playwright_crawler_stagehand/support_classes.py';
+import BrowserClasses from '!!raw-loader!./code_examples/playwright_crawler_stagehand/browser_classes.py';
+import StagehandRun from '!!raw-loader!./code_examples/playwright_crawler_stagehand/stagehand_run.py';
+
+[Stagehand](https://docs.stagehand.dev/) is a framework that combines [Playwright](https://playwright.dev/python/) with AI-driven natural language understanding and decision-making capabilities. With Stagehand, you can use natural language instructions to interact with web pages instead of writing complex selectors and automation logic.
+
+Stagehand supports multiple AI models through [`LiteLLM`](https://docs.litellm.ai/docs/). This guide demonstrates how to integrate Stagehand with <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> using [Gemini](https://ai.google.dev/gemini-api/docs) as the AI model provider.
+
+:::info
+
+This guide is based on stagehand-python v0.4.0 with local configuration settings and may not be compatible with newer versions.
+
+:::
+
+## Get Gemini API key
+
+You need to register with [Google AI Studio](https://aistudio.google.com/) and navigate to [Get API key](https://aistudio.google.com/app/apikey) to obtain your API key.
+
+## Create support classes for Stagehand
+
+To integrate Stagehand with Crawlee, you need to create wrapper classes that allow <ApiLink to="class/PlaywrightBrowserPlugin">`PlaywrightBrowserPlugin`</ApiLink> to manage the Playwright lifecycle.
+
+Create `CrawleeStagehand` - a custom Stagehand subclass that overrides the `init` method to prevent Stagehand from launching its own Playwright instance.
+
+Create `CrawleeStagehandPage` - a wrapper class for `StagehandPage` that implements the [Playwright Page](https://playwright.dev/python/docs/next/api/class-page) behavior expected by <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink>.
+
+<CodeBlock className="language-python" title="support_classes.py">
+    {SupportClasses}
+</CodeBlock>
+
+## Create browser integration classes
+
+You need to create a custom browser plugin and controller that properly initialize Stagehand and obtain browser pages from `StagehandContext`.
+
+Create `StagehandPlugin` - a subclass of <ApiLink to="class/PlaywrightBrowserPlugin">`PlaywrightBrowserPlugin`</ApiLink> that holds the Stagehand instance and creates `PlaywrightPersistentBrowser` instances.
+
+Create `StagehandBrowserController` - a subclass of <ApiLink to="class/PlaywrightBrowserController">`PlaywrightBrowserController`</ApiLink> that lazily initializes `StagehandContext` and creates new pages with AI capabilities on demand.
+
+<CodeBlock className="language-python" title="browser_classes.py">
+    {BrowserClasses}
+</CodeBlock>
+
+## Create a crawler
+
+Now you can create a <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> that uses Stagehand's AI capabilities to interact with web pages using natural language commands:
+
+<CodeBlock className="language-python" title="stagehand_run.py">
+    {StagehandRun}
+</CodeBlock>
+
+The integration works through several key components:
+- `CrawleeStagehand` prevents Stagehand from launching its own Playwright instance, allowing Crawlee to manage the browser lifecycle
+- `StagehandPlugin` extends the Playwright browser plugin to create Stagehand-enabled browser instances
+- `StagehandBrowserController` uses `StagehandContext` to create pages with AI capabilities
+- `CrawleeStagehandPage` provides interface compatibility between Stagehand pages and Crawlee's expectations
+
+In the request handler, you can use natural language commands like `page.extract('Extract title page')` to perform intelligent data extraction without writing complex selectors.

docs/guides/playwright_crawler_stagehand.mdx

@@ -239,6 +239,7 @@ module = [
     "apify_fingerprint_datapoints", # Untyped and stubs not available
     "camoufox",                     # Example code shows integration of camoufox and crawlee.
     "fastapi",                      # Example code shows running in webserver.
+    "stagehand.*",                  # Example code shows integration of Stagehand and crawlee.
     "starlette.*",                  # Example code shows running in webserver.
     "flask",                        # Example code shows deploy on Google Cloud.
     "functions_framework",          # Example code shows deploy on Google Cloud.