add CDP

Author: loorisr

.env.example

@@ -7,4 +7,5 @@ PROXY_SERVER=
 PROXY_USERNAME=
 PROXY_PASSWORD=
 PORT=3000
-PERSISTENT_CONTEXT=False
+PERSISTENT_CONTEXT=False
+REMOTE_CDP=

Dockerfile

@@ -1,9 +1,9 @@
 FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim
 
 LABEL maintainer="loorisr"
-LABEL repository="https://github.com/loorisr/playwright-scrape-api"
+LABEL repository="https://github.com/loorisr/patchright-scrape-api"
 LABEL description="Simple scraping API based on patchright "
-LABEL date="2025-02-24"
+LABEL date="2025-02-26"
 
 # Install the project into `/app`
 WORKDIR /app

README.md

@@ -13,8 +13,9 @@ Features:
 * better media blocking handling
 * scrape multiple pages in parallel
 * scrape endpoint compatible with Firecrawl API
-* return cleaned html and markdow
+* return cleaned html and markdown
 * temporary or persistent context
+* can connect to remote browser via CDP
 * lightweight: 1.2 Go
 
 Available on Docker hub: `docker pull loorisr/patchright-scrape-api:latest`
@@ -38,6 +39,8 @@ Available on Docker hub: `docker pull loorisr/patchright-scrape-api:latest`
 
 * `PERSISTENT_CONTEXT`: To enable persistent context. If true, a volume needs to be mounted at /context. Default: False
 
+* `REMOTE_CDP`: Address of a remote browser with CDP (Chrome DevTools Protocol). Allows you to connect to a provider or use https://github.com/JacobLinCool/playwright-docker for example.
+
 ## Endpoints
 * `/scrape`
   - **url**: url to scrape : http://www.domain.tld

app/app.py

@@ -45,6 +45,8 @@
 PERSISTENT_CONTEXT = os.getenv("PERSISTENT_CONTEXT", 'False').lower() in ('true', '1', 't')
 PATH_CONTEXT = "/context"
 
+REMOTE_CDP = os.getenv('REMOTE_CDP')
+
 # Global browser and context instances
 browser: Browser = None
 context: BrowserContext = None
@@ -170,46 +172,50 @@ async def lifespan(app: FastAPI):
     # Startup logic
     playwright = await async_playwright().start()
 
-    if PERSISTENT_CONTEXT:
-        print("Launching Chrome with persistent context.")
-        context = await playwright.chromium.launch_persistent_context(
-            user_data_dir=PATH_CONTEXT,
-            headless=True,
-            channel="chrome",
-            args=[
-                '--no-sandbox',
-                '--disable-setuid-sandbox',
-                '--disable-dev-shm-usage',
-                '--disable-accelerated-2d-canvas',
-                '--no-first-run',
-                '--no-zygote',
-                '--single-process',
-                '--disable-gpu',
-                '--no-default-browser-check',
-                '--disable-infobars'
-            ],
-            **context_options
-        )
+    if REMOTE_CDP:
+        browser = await playwright.chromium.connect_over_cdp(f"wss://{REMOTE_CDP}")
+        context = browser.contexts[0]
     else:
-        print("Launching Chrome with temporary context.")
-        browser = await playwright.chromium.launch(
-            headless=True,
-            channel="chrome",
-            args=[
-                '--no-sandbox',
-                '--disable-setuid-sandbox',
-                '--disable-dev-shm-usage',
-                '--disable-accelerated-2d-canvas',
-                '--no-first-run',
-                '--no-zygote',
-                '--single-process',
-                '--disable-gpu',
-                '--no-default-browser-check',
-                '--no-startup-window',
-                '--disable-infobars'
-            ]
-        )
-        context = await browser.new_context(**context_options)
+        if PERSISTENT_CONTEXT:
+            print("Launching Chrome with persistent context.")
+            context = await playwright.chromium.launch_persistent_context(
+                user_data_dir=PATH_CONTEXT,
+                headless=True,
+                channel="chrome",
+                args=[
+                    '--no-sandbox',
+                    '--disable-setuid-sandbox',
+                    '--disable-dev-shm-usage',
+                    '--disable-accelerated-2d-canvas',
+                    '--no-first-run',
+                    '--no-zygote',
+                    '--single-process',
+                    '--disable-gpu',
+                    '--no-default-browser-check',
+                    '--disable-infobars'
+                ],
+                **context_options
+            )
+        else:
+            print("Launching Chrome with temporary context.")
+            browser = await playwright.chromium.launch(
+                headless=True,
+                channel="chrome",
+                args=[
+                    '--no-sandbox',
+                    '--disable-setuid-sandbox',
+                    '--disable-dev-shm-usage',
+                    '--disable-accelerated-2d-canvas',
+                    '--no-first-run',
+                    '--no-zygote',
+                    '--single-process',
+                    '--disable-gpu',
+                    '--no-default-browser-check',
+                    '--no-startup-window',
+                    '--disable-infobars'
+                ]
+            )
+            context = await browser.new_context(**context_options)
 
 
     if RESOURCES_BLOCKED:

docker-compose.yaml

@@ -14,6 +14,7 @@ services:
       - PROXY_PASSWORD=
       - PORT=3000
       - PERSISTENT_CONTEXT=False
+      - REMOTE_CDP=
     volumes:
       - patchright_context:/context # needed if PERSISTENT_CONTEXT=True
     restart: unless-stopped