Add a crawler task using crawl4ai

Author: ericallam

apps/webapp/app/components/runs/v3/RunIcon.tsx

@@ -73,7 +73,7 @@ export function RunIcon({ name, className, spanName }: TaskIconProps) {
     case "trigger":
       return <TriggerIcon className={cn(className, "text-orange-500")} />;
     case "python":
-      return <PythonLogoIcon />;
+      return <PythonLogoIcon className={className} />;
     //log levels
     case "debug":
     case "log":

pnpm-lock.yaml

@@ -8,7 +8,14 @@
     "start": "next start",
     "lint": "next lint",
     "dev:trigger": "trigger dev",
-    "tunnel": "ngrok http --url=d3-demo.ngrok.dev 3000"
+    "tunnel": "ngrok http --url=d3-demo.ngrok.dev 3000",
+    "python:install-requirements": "uv pip sync requirements.txt",
+    "python:compile-requirements": "uv pip compile requirements.in -o requirements.txt",
+    "python:install-browsers": "./.venv/bin/playwright install",
+    "python:install": "pnpm run python:compile-requirements && pnpm run python:install-requirements",
+    "python:create-env": "uv venv .venv",
+    "db:migrate": "tsx -r dotenv/config src/lib/migrate.ts up",
+    "db:migrate:down": "tsx -r dotenv/config src/lib/migrate.ts down"
   },
   "dependencies": {
     "@radix-ui/react-avatar": "^1.1.3",
@@ -23,6 +30,7 @@
     "zod": "3.23.8",
     "@trigger.dev/react-hooks": "workspace:*",
     "@trigger.dev/sdk": "workspace:*",
+    "@trigger.dev/python": "workspace:*",
     "nanoid": "^5.1.5",
     "ai": "4.2.5",
     "@ai-sdk/openai": "1.3.3",

references/d3-chat/package.json

@@ -0,0 +1,2 @@
+crawl4ai
+playwright

references/d3-chat/requirements.in

@@ -0,0 +1,218 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile requirements.in -o requirements.txt
+aiofiles==24.1.0
+    # via crawl4ai
+aiohappyeyeballs==2.6.1
+    # via aiohttp
+aiohttp==3.11.15
+    # via
+    #   crawl4ai
+    #   litellm
+aiosignal==1.3.2
+    # via aiohttp
+aiosqlite==0.21.0
+    # via crawl4ai
+annotated-types==0.7.0
+    # via pydantic
+anyio==4.9.0
+    # via
+    #   httpx
+    #   openai
+attrs==25.3.0
+    # via
+    #   aiohttp
+    #   jsonschema
+    #   referencing
+beautifulsoup4==4.13.3
+    # via crawl4ai
+certifi==2025.1.31
+    # via
+    #   httpcore
+    #   httpx
+    #   requests
+cffi==1.17.1
+    # via cryptography
+charset-normalizer==3.4.1
+    # via requests
+click==8.1.8
+    # via
+    #   crawl4ai
+    #   litellm
+    #   nltk
+colorama==0.4.6
+    # via crawl4ai
+crawl4ai==0.5.0.post8
+    # via -r requirements.in
+cryptography==44.0.2
+    # via pyopenssl
+cssselect==1.3.0
+    # via crawl4ai
+distro==1.9.0
+    # via openai
+fake-http-header==0.3.5
+    # via tf-playwright-stealth
+fake-useragent==2.1.0
+    # via crawl4ai
+faust-cchardet==2.1.19
+    # via crawl4ai
+filelock==3.18.0
+    # via huggingface-hub
+frozenlist==1.5.0
+    # via
+    #   aiohttp
+    #   aiosignal
+fsspec==2025.3.2
+    # via huggingface-hub
+greenlet==3.1.1
+    # via playwright
+h11==0.14.0
+    # via httpcore
+httpcore==1.0.7
+    # via httpx
+httpx==0.28.1
+    # via
+    #   crawl4ai
+    #   litellm
+    #   openai
+huggingface-hub==0.30.1
+    # via tokenizers
+humanize==4.12.2
+    # via crawl4ai
+idna==3.10
+    # via
+    #   anyio
+    #   httpx
+    #   requests
+    #   yarl
+importlib-metadata==8.6.1
+    # via litellm
+jinja2==3.1.6
+    # via litellm
+jiter==0.9.0
+    # via openai
+joblib==1.4.2
+    # via nltk
+jsonschema==4.23.0
+    # via litellm
+jsonschema-specifications==2024.10.1
+    # via jsonschema
+litellm==1.65.1
+    # via crawl4ai
+lxml==5.3.1
+    # via crawl4ai
+markdown-it-py==3.0.0
+    # via rich
+markupsafe==3.0.2
+    # via jinja2
+mdurl==0.1.2
+    # via markdown-it-py
+multidict==6.3.0
+    # via
+    #   aiohttp
+    #   yarl
+nltk==3.9.1
+    # via crawl4ai
+numpy==2.2.4
+    # via
+    #   crawl4ai
+    #   rank-bm25
+openai==1.70.0
+    # via litellm
+packaging==24.2
+    # via huggingface-hub
+pillow==10.4.0
+    # via crawl4ai
+playwright==1.51.0
+    # via
+    #   -r requirements.in
+    #   crawl4ai
+    #   tf-playwright-stealth
+propcache==0.3.1
+    # via
+    #   aiohttp
+    #   yarl
+psutil==7.0.0
+    # via crawl4ai
+pycparser==2.22
+    # via cffi
+pydantic==2.11.1
+    # via
+    #   crawl4ai
+    #   litellm
+    #   openai
+pydantic-core==2.33.0
+    # via pydantic
+pyee==12.1.1
+    # via playwright
+pygments==2.19.1
+    # via rich
+pyopenssl==25.0.0
+    # via crawl4ai
+pyperclip==1.9.0
+    # via crawl4ai
+python-dotenv==1.1.0
+    # via
+    #   crawl4ai
+    #   litellm
+pyyaml==6.0.2
+    # via huggingface-hub
+rank-bm25==0.2.2
+    # via crawl4ai
+referencing==0.36.2
+    # via
+    #   jsonschema
+    #   jsonschema-specifications
+regex==2024.11.6
+    # via
+    #   nltk
+    #   tiktoken
+requests==2.32.3
+    # via
+    #   crawl4ai
+    #   huggingface-hub
+    #   tiktoken
+rich==14.0.0
+    # via crawl4ai
+rpds-py==0.24.0
+    # via
+    #   jsonschema
+    #   referencing
+sniffio==1.3.1
+    # via
+    #   anyio
+    #   openai
+snowballstemmer==2.2.0
+    # via crawl4ai
+soupsieve==2.6
+    # via beautifulsoup4
+tf-playwright-stealth==1.1.2
+    # via crawl4ai
+tiktoken==0.9.0
+    # via litellm
+tokenizers==0.21.1
+    # via litellm
+tqdm==4.67.1
+    # via
+    #   huggingface-hub
+    #   nltk
+    #   openai
+typing-extensions==4.13.0
+    # via
+    #   aiosqlite
+    #   beautifulsoup4
+    #   huggingface-hub
+    #   openai
+    #   pydantic
+    #   pydantic-core
+    #   pyee
+    #   typing-inspection
+typing-inspection==0.4.0
+    # via pydantic
+urllib3==2.3.0
+    # via requests
+xxhash==3.5.0
+    # via crawl4ai
+yarl==1.18.3
+    # via aiohttp
+zipp==3.21.0
+    # via importlib-metadata

references/d3-chat/requirements.txt

@@ -0,0 +1,36 @@
+import type { BuildContext, BuildExtension } from "@trigger.dev/build";
+
+// This is a custom build extension to install Playwright and Chromium
+export function installPlaywrightChromium(): BuildExtension {
+  return {
+    name: "InstallPlaywrightChromium",
+    onBuildComplete(context: BuildContext) {
+      const instructions = [
+        // Base and Chromium dependencies
+        `RUN apt-get update && apt-get install -y --no-install-recommends \
+          curl unzip npm libnspr4 libatk1.0-0 libatk-bridge2.0-0 libatspi2.0-0 \
+          libasound2 libnss3 libxcomposite1 libxdamage1 libxfixes3 libxrandr2 \
+          libgbm1 libxkbcommon0 \
+          && apt-get clean && rm -rf /var/lib/apt/lists/*`,
+
+        // Install Playwright and Chromium
+        `RUN npm install -g playwright`,
+        `RUN mkdir -p /ms-playwright`,
+        `RUN PLAYWRIGHT_BROWSERS_PATH=/ms-playwright python -m playwright install --with-deps chromium`,
+      ];
+
+      context.addLayer({
+        id: "playwright",
+        image: { instructions },
+        deploy: {
+          env: {
+            PLAYWRIGHT_BROWSERS_PATH: "/ms-playwright",
+            PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "1",
+            PLAYWRIGHT_SKIP_BROWSER_VALIDATION: "1",
+          },
+          override: true,
+        },
+      });
+    },
+  };
+}

references/d3-chat/src/extensions/playwright.ts

@@ -0,0 +1,49 @@
+import { sql } from "@vercel/postgres";
+
+export async function migrate(direction: "up" | "down") {
+  if (direction === "up") {
+    await migrateUp();
+  } else {
+    await migrateDown();
+  }
+}
+
+export async function migrateUp() {
+  const createTable = await sql`
+    CREATE TABLE IF NOT EXISTS todos (
+      id VARCHAR(255) PRIMARY KEY,
+      user_id VARCHAR(255) NOT NULL,
+      title VARCHAR(255) NOT NULL,
+      description TEXT,
+      status VARCHAR(50) NOT NULL DEFAULT 'pending',
+      priority INTEGER NOT NULL DEFAULT 3,
+      due_date TIMESTAMP WITH TIME ZONE,
+      created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+      updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+      completed_at TIMESTAMP WITH TIME ZONE,
+      tags TEXT[], -- Array of tags
+      assigned_to VARCHAR(255)
+    );
+  `;
+
+  console.log(`Created "todos" table`);
+
+  return {
+    createTable,
+  };
+}
+
+export async function migrateDown() {
+  const dropTable = await sql`
+    DROP TABLE IF EXISTS todos;
+  `;
+
+  console.log(`Dropped "todos" table`);
+}
+
+async function main() {
+  const direction = process.argv[2];
+  await migrate(direction as "up" | "down");
+}
+
+main().catch(console.error);