Merge pull request #424 from trycompai/lewis/vendor-research

feat: integrate Firecrawl for vendor research and add global vendor table

Author: carhartlewis

apps/app/package.json

@@ -31,6 +31,7 @@
 		"@dnd-kit/sortable": "^10.0.0",
 		"@dnd-kit/utilities": "^3.2.2",
 		"@hookform/resolvers": "^3.10.0",
+		"@mendable/firecrawl-js": "^1.24.0",
 		"@nangohq/frontend": "^0.53.2",
 		"@next/third-parties": "^15.3.0",
 		"@novu/headless": "^2.6.6",

apps/app/src/actions/research-vendor.ts

@@ -0,0 +1,131 @@
+"use server";
+import { type ActionResponse } from "@/app/actions/actions";
+import { env } from "@/env.mjs";
+import { db } from "@comp/db";
+import FirecrawlApp, { ScrapeResponse } from "@mendable/firecrawl-js";
+import ky from "ky";
+import { z } from "zod";
+import { authActionClient } from "./safe-action";
+
+let firecrawl: FirecrawlApp | null = null;
+
+if (process.env.FIRECRAWL_API_KEY) {
+	firecrawl = new FirecrawlApp({
+		apiKey: process.env.FIRECRAWL_API_KEY,
+	});
+}
+
+const schema = z.object({
+	company_name: z.string(),
+	legal_name: z.string(),
+	company_description: z.string(),
+	company_hq_address: z.string(),
+	privacy_policy_url: z.string(),
+	terms_of_service_url: z.string(),
+	service_level_agreement_url: z.string(),
+	security_overview_url: z.string(),
+	trust_portal_url: z.string(),
+	certified_security_frameworks: z.array(z.string()),
+	subprocessors: z.array(z.string()),
+	type_of_company: z.string(),
+});
+
+export const researchVendorAction = authActionClient
+	.schema(
+		z.object({
+			website: z.string().url({ message: "Invalid URL format" }),
+		}),
+	)
+	.metadata({
+		name: "research-vendor",
+	})
+	.action(async ({ parsedInput: { website }, ctx: { user } }) => {
+		try {
+			if (!firecrawl) {
+				return {
+					success: false,
+					error: { message: "Firecrawl client not initialized" },
+				};
+			}
+
+			const existingVendor = await db.globalVendors.findUnique({
+				where: {
+					website: website,
+				},
+				select: { website: true },
+			});
+
+			if (existingVendor) {
+				return {
+					success: true,
+					data: {
+						message: "Vendor already exists.",
+						vendorExists: true,
+					},
+				};
+			}
+
+			const scrapeResult = await firecrawl.extract([website], {
+				prompt: "You're a cyber security researcher, researching a vendor.",
+				schema: schema,
+				enableWebSearch: true,
+				scrapeOptions: {
+					onlyMainContent: true,
+					removeBase64Images: true,
+				},
+			});
+
+			if (!scrapeResult.success || !scrapeResult.data) {
+				return {
+					success: false,
+					error: {
+						message: `Failed to scrape vendor data: ${
+							scrapeResult.error || "Unknown error"
+						}`,
+					},
+				};
+			}
+
+			const vendorData = scrapeResult.data as z.infer<typeof schema>;
+
+			await db.globalVendors.create({
+				data: {
+					website: website,
+					company_name: vendorData.company_name,
+					legal_name: vendorData.legal_name,
+					company_description: vendorData.company_description,
+					company_hq_address: vendorData.company_hq_address,
+					privacy_policy_url: vendorData.privacy_policy_url,
+					terms_of_service_url: vendorData.terms_of_service_url,
+					service_level_agreement_url:
+						vendorData.service_level_agreement_url,
+					security_page_url: vendorData.security_overview_url,
+					trust_page_url: vendorData.trust_portal_url,
+					security_certifications:
+						vendorData.certified_security_frameworks,
+					subprocessors: vendorData.subprocessors,
+					type_of_company: vendorData.type_of_company,
+				},
+			});
+
+			return {
+				success: true,
+				data: {
+					message: "Vendor researched and added successfully.",
+					vendorExists: false,
+				},
+			};
+		} catch (error) {
+			console.error("Error in researchVendorAction:", error);
+
+			return {
+				success: false,
+				error: {
+					message:
+						error instanceof Error
+							? error.message
+							: "An unexpected error occurred.",
+				},
+			};
+		}
+	});

apps/app/src/app/[locale]/(app)/(dashboard)/[orgId]/vendors/components/create-vendor-form.tsx

@@ -1,5 +1,6 @@
 "use client";
 
+import { researchVendorAction } from "@/actions/research-vendor";
 import { SelectAssignee } from "@/components/SelectAssignee";
 import { useI18n } from "@/locales/client";
 import { Member, User, VendorCategory, VendorStatus } from "@comp/db/types";
@@ -28,6 +29,7 @@ import {
 } from "@comp/ui/select";
 import { Textarea } from "@comp/ui/textarea";
 import { zodResolver } from "@hookform/resolvers/zod";
+import ky from "ky";
 import { ArrowRightIcon } from "lucide-react";
 import { useAction } from "next-safe-action/hooks";
 import { useQueryState } from "nuqs";
@@ -52,7 +54,7 @@ export function CreateVendorForm({
 	const [_, setCreateVendorSheet] = useQueryState("createVendorSheet");
 
 	const createVendor = useAction(createVendorAction, {
-		onSuccess: async (data) => {
+		onSuccess: async () => {
 			toast.success(t("vendors.form.create_vendor_success"));
 			setCreateVendorSheet(null);
 		},
@@ -61,6 +63,8 @@ export function CreateVendorForm({
 		},
 	});
 
+	const researchVendor = useAction(researchVendorAction);
+
 	const form = useForm<z.infer<typeof createVendorSchema>>({
 		resolver: zodResolver(createVendorSchema),
 		defaultValues: {
@@ -71,8 +75,14 @@ export function CreateVendorForm({
 		},
 	});
 
-	const onSubmit = (data: z.infer<typeof createVendorSchema>) => {
+	const onSubmit = async (data: z.infer<typeof createVendorSchema>) => {
 		createVendor.execute(data);
+
+		if (data.website) {
+			researchVendor.execute({
+				website: data.website,
+			});
+		}
 	};
 
 	return (

apps/app/src/app/api/chat/route.ts

@@ -1,7 +1,7 @@
 import { tools } from "@/data/tools";
 import { model, type modelID } from "@/hooks/ai/providers";
 import { auth } from "@/utils/auth";
-import { type UIMessage, streamText } from "ai";
+import { type UIMessage, smoothStream, streamText } from "ai";
 import { headers } from "next/headers";
 
 export const maxDuration = 30;

apps/app/src/instrumentation-client.ts

@@ -24,3 +24,5 @@ Sentry.init({
 	// Setting this option to true will print useful information to the console while you're setting up Sentry.
 	debug: false,
 });
+
+export const onRouterTransitionStart = Sentry.captureRouterTransitionStart;

bun.lock

@@ -0,0 +1,26 @@
+-- CreateTable
+CREATE TABLE "GlobalVendors" (
+    "website" TEXT NOT NULL,
+    "company_name" TEXT,
+    "legal_name" TEXT,
+    "company_description" TEXT,
+    "company_hq_address" TEXT,
+    "privacy_policy_url" TEXT,
+    "terms_of_service_url" TEXT,
+    "service_level_agreement_url" TEXT,
+    "security_page_url" TEXT,
+    "trust_page_url" TEXT,
+    "security_certifications" TEXT[],
+    "subprocessors" TEXT[],
+    "type_of_company" TEXT,
+    "approved" BOOLEAN NOT NULL DEFAULT false,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "GlobalVendors_pkey" PRIMARY KEY ("website")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "GlobalVendors_website_key" ON "GlobalVendors"("website");
+
+-- CreateIndex
+CREATE INDEX "GlobalVendors_website_idx" ON "GlobalVendors"("website");

packages/db/prisma/migrations/20250417205622_add_global_vendors/migration.sql

@@ -1,3 +1,3 @@
 # Please do not edit this file manually
 # It should be added in your version-control system (e.g., Git)
-provider = "postgresql"
+provider = "postgresql"

packages/db/prisma/migrations/migration_lock.toml

@@ -29,6 +29,28 @@ model AuditLog {
     @@index([organizationId])
 }
 
+model GlobalVendors {
+    website String @id @unique
+    company_name String?
+    legal_name String?
+    company_description String?
+    company_hq_address String?
+    privacy_policy_url String?
+    terms_of_service_url String?
+    service_level_agreement_url String?
+    security_page_url String?
+    trust_page_url String?
+    security_certifications String[]
+    subprocessors String[]
+    type_of_company String?
+
+    approved Boolean @default(false)
+    createdAt DateTime @default(now())
+
+
+    @@index([website])
+}
+
 enum Departments {
     none
     admin

packages/db/prisma/schema/shared.prisma

@@ -42,7 +42,8 @@
         "REVALIDATION_SECRET",
         "GROQ_API_KEY",
         "SENTRY_AUTH_TOKEN",
-        "RESEND_AUDIENCE_ID"
+        "RESEND_AUDIENCE_ID",
+        "FIRECRAWL_API_KEY"
       ],
       "inputs": ["$TURBO_DEFAULT$", ".env"],
       "dependsOn": ["^build", "^db:generate", "^auth:build", "clean-react"],