feat: Add example of how to integrate Camoufox into PlaywrightCrawler (#789)

### Description

Show how to integrate `Camoufox` into `PlaywrightCrawler`. 
Added one docs page and one example code.
Added `playwright-camoufox` template to cli call `crawlee create`.
Fixed wrong template indentation when `Apify integration` was selected
in cli.

### Issues

- Closes: #684

Author: Pijukatel

docs/examples/code/playwright_crawler_with_camoufox.py

@@ -0,0 +1,60 @@
+import asyncio
+
+#  Camoufox is external package and needs to be installed. It is not included in crawlee.
+from camoufox import AsyncNewBrowser
+from typing_extensions import override
+
+from crawlee.browsers import BrowserPool, PlaywrightBrowserController, PlaywrightBrowserPlugin
+from crawlee.playwright_crawler import PlaywrightCrawler, PlaywrightCrawlingContext
+
+
+class CamoufoxPlugin(PlaywrightBrowserPlugin):
+    """Example browser plugin that uses Camoufox browser, but otherwise keeps the functionality of
+    PlaywrightBrowserPlugin."""
+
+    @override
+    async def new_browser(self) -> PlaywrightBrowserController:
+        if not self._playwright:
+            raise RuntimeError('Playwright browser plugin is not initialized.')
+
+        return PlaywrightBrowserController(
+            browser=await AsyncNewBrowser(self._playwright, headless=True, **self._browser_options),
+            max_open_pages_per_browser=1,  # Increase, if camoufox can handle it in your use case.
+            header_generator=None,  # This turns off the crawlee header_generation. Camoufox has its own.
+        )
+
+
+async def main() -> None:
+    crawler = PlaywrightCrawler(
+        # Limit the crawl to max requests. Remove or increase it for crawling all links.
+        max_requests_per_crawl=10,
+        # Custom browser pool. This gives users full control over browsers used by the crawler.
+        browser_pool=BrowserPool(plugins=[CamoufoxPlugin()]),
+    )
+
+    # Define the default request handler, which will be called for every request.
+    @crawler.router.default_handler
+    async def request_handler(context: PlaywrightCrawlingContext) -> None:
+        context.log.info(f'Processing {context.request.url} ...')
+
+        # Extract some data from the page using Playwright's API.
+        posts = await context.page.query_selector_all('.athing')
+        for post in posts:
+            # Get the HTML elements for the title and rank within each post.
+            title_element = await post.query_selector('.title a')
+
+            # Extract the data we want from the elements.
+            title = await title_element.inner_text() if title_element else None
+
+        # Push the extracted data to the default dataset.
+        await context.push_data({'title': title})
+
+        # Find a link to the next page and enqueue it if it exists.
+        await context.enqueue_links(selector='.morelink')
+
+    # Run the crawler with the initial list of URLs.
+    await crawler.run(['https://news.ycombinator.com/'])
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

docs/examples/playwright_crawler_with_camoufox.mdx

@@ -0,0 +1,26 @@
+---
+id: playwright-crawler-with-camoufox
+title: Playwright crawler with Camoufox
+---
+
+import ApiLink from '@site/src/components/ApiLink';
+import CodeBlock from '@theme/CodeBlock';
+
+import PlaywrightCrawlerExampleWithCamoufox from '!!raw-loader!./code/playwright_crawler_with_camoufox.py';
+
+This example demonstrates how to integrate Camoufox into <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> using <ApiLink to="class/BrowserPool">`BrowserPool`</ApiLink> with custom <ApiLink to="class/PlaywrightBrowserPlugin">`PlaywrightBrowserPlugin`</ApiLink>.
+
+Camoufox is a stealthy minimalistic build of Firefox. For details please visit its homepage https://camoufox.com/ .
+To be able to run this example you will need to install camoufox, as it is external tool, and it is not part of the crawlee. For installation please see https://pypi.org/project/camoufox/.
+
+**Warning!** Camoufox is using custom build of firefox. This build can be hundreds of MB large.
+You can either pre-download this file using following command `python3 -m camoufox fetch` or camoufox will download it automatically once you try to run it, and it does not find existing binary.
+For more details please refer to: https://github.com/daijro/camoufox/tree/main/pythonlib#camoufox-python-interface
+
+**Project template -** It is possible to generate project with Python code which includes Camoufox integration into crawlee through crawlee cli. Call `crawlee create` and pick `Playwright-camoufox` when asked for Crawler type.
+
+The example code after PlayWrightCrawler instantiation is similar to example describing the use of Playwright Crawler. The main difference is that in this example Camoufox will be used as the browser through BrowserPool.
+
+<CodeBlock className="language-python">
+    {PlaywrightCrawlerExampleWithCamoufox}
+</CodeBlock>

pyproject.toml

@@ -205,8 +205,11 @@ warn_unreachable = true
 warn_unused_ignores = true
 
 [[tool.mypy.overrides]]
-# Example code that shows integration of crawlee imports apify, despite apify not being dependency of crawlee.
-module = "apify"
+# Example codes are sometimes showing integration of crawlee with external tool, which is not dependency of crawlee.
+module =[
+    "apify",  # Example code shows integration of apify and crawlee.
+    "camoufox"  # Example code shows integration of camoufox and crawlee.
+]
 ignore_missing_imports = true
 
 [tool.basedpyright]

src/crawlee/project_template/cookiecutter.json

@@ -1,7 +1,8 @@
 {
     "project_name": "crawlee-python-project",
     "__package_name": "{{ cookiecutter.project_name|lower|replace('-', '_') }}",
-    "crawler_type": ["beautifulsoup", "parsel", "playwright"],
+    "crawler_type": ["beautifulsoup", "parsel", "playwright", "playwright-camoufox"],
+    "__crawler_type": "{{ cookiecutter.crawler_type|lower|replace('-', '_') }}",
     "http_client": ["httpx", "curl-impersonate"],
     "package_manager": ["poetry", "pip", "manual"],
     "enable_apify_integration": false,

src/crawlee/project_template/templates/main.py

@@ -20,7 +20,11 @@
 # % endif
 # % endblock
 # % endfilter
+# % if self.pre_main is defined
 
+{{self.pre_main()}}
+
+# % endif
 async def main() -> None:
     """The crawler entry point."""
     # % filter truncate(0, end='')
@@ -30,17 +34,16 @@ async def main() -> None:
 
     # % if cookiecutter.enable_apify_integration
     async with Actor:
-        # % filter indent(width=8, first=False)
-        {{ self.instantiation() }}
-        # % endfilter
+    # % set indent_width = 8
     # % else
-        # % filter indent(width=4, first=False)
-    {{ self.instantiation() }}
-        # % endfilter
+    # % set indent_width = 4
     # % endif
+# % filter indent(width=indent_width, first=True)
+{{self.instantiation()}}
 
-    await crawler.run(
-        [
-            '{{ cookiecutter.start_url }}',
-        ]
-    )
+await crawler.run(
+    [
+        '{{ cookiecutter.start_url }}',
+    ]
+)
+# % endfilter

src/crawlee/project_template/templates/main_playwright_camoufox.py

@@ -0,0 +1,36 @@
+# % extends 'main.py'
+
+# % block import
+from camoufox import AsyncNewBrowser
+from typing_extensions import override
+
+from crawlee._utils.context import ensure_context
+from crawlee.browsers import PlaywrightBrowserPlugin, PlaywrightBrowserController, BrowserPool
+from crawlee.playwright_crawler import PlaywrightCrawler
+# % endblock
+
+# % block pre_main
+class CamoufoxPlugin(PlaywrightBrowserPlugin):
+    """Example browser plugin that uses Camoufox Browser, but otherwise keeps the functionality of
+    PlaywrightBrowserPlugin."""
+
+    @ensure_context
+    @override
+    async def new_browser(self) -> PlaywrightBrowserController:
+        if not self._playwright:
+            raise RuntimeError('Playwright browser plugin is not initialized.')
+
+        return PlaywrightBrowserController(
+            browser=await AsyncNewBrowser(self._playwright, headless=True),
+            max_open_pages_per_browser=1,  #  Increase, if camoufox can handle it in your usecase.
+            header_generator=None,  #  This turns off the crawlee header_generation. Camoufox has its own.
+        )
+# % endblock
+
+# % block instantiation
+crawler = PlaywrightCrawler(
+    max_requests_per_crawl=10,
+    request_handler=router,
+    browser_pool=BrowserPool(plugins=[CamoufoxPlugin()])
+)
+# % endblock

src/crawlee/project_template/templates/routes_playwright.py renamed to src/crawlee/project_template/templates/routes_camoufox.py

@@ -0,0 +1,19 @@
+from crawlee.playwright_crawler import PlaywrightCrawlingContext
+from crawlee.router import Router
+
+router = Router[PlaywrightCrawlingContext]()
+
+
+@router.default_handler
+async def default_handler(context: PlaywrightCrawlingContext) -> None:
+    """Default request handler."""
+    context.log.info(f'Processing {context.request.url} ...')
+    title = await context.page.query_selector('title')
+    await context.push_data(
+        {
+            'url': context.request.loaded_url,
+            'title': await title.inner_text() if title else None,
+        }
+    )
+
+    await context.enqueue_links()

src/crawlee/project_template/templates/routes_playwright_camoufox.py

@@ -3,6 +3,9 @@
 # You can also use any other image from Docker Hub.
 # % if cookiecutter.crawler_type == 'playwright'
 FROM apify/actor-python-playwright:3.13
+# % elif cookiecutter.crawler_type == 'camoufox'
+# Currently camoufox has issues installing on Python 3.13
+FROM apify/actor-python-playwright:3.12
 # % else
 FROM apify/actor-python:3.13
 # % endif

src/crawlee/project_template/{{cookiecutter.project_name}}/Dockerfile

@@ -1,4 +1,8 @@
+# % if cookiecutter.crawler_type == 'camoufox'
+# % set extras = ['playwright']
+# % else
 # % set extras = [cookiecutter.crawler_type]
+# % endif
 # % if cookiecutter.http_client == 'curl-impersonate'
 # % do extras.append('curl-impersonate')
 # % endif
@@ -14,6 +18,9 @@ package-mode = false
 
 [tool.poetry.dependencies]
 python = "^3.9"
+# % if cookiecutter.crawler_type == 'camoufox'
+camoufox = {version= ">=0.4.5", extras = ["geoip"]}
+# % endif
 # % if cookiecutter.enable_apify_integration
 apify = "*"
 # % endif