diff --git a/.gitignore b/.gitignore index aab9ecc..165e1ea 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,8 @@ yarn-error.log* pnpm-debug.log* /dist /.turbo +.vscode/ +.idea/ +*.swp +*.swo +*~ \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..6b0e5ab --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "postman.settings.dotenv-detection-notification-visibility": false +} \ No newline at end of file diff --git a/examples/computer-use-nextjs/.gitignore b/examples/computer-use-nextjs/.gitignore new file mode 100644 index 0000000..bfe57c0 --- /dev/null +++ b/examples/computer-use-nextjs/.gitignore @@ -0,0 +1,42 @@ +# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. + +# dependencies +/node_modules +/.pnp +.pnp.* +.yarn/* +!.yarn/patches +!.yarn/plugins +!.yarn/releases +!.yarn/versions + +# testing +/coverage + +# next.js +/.next/ +/out/ + +# production +/build + +# misc +.DS_Store +*.pem + +# debug +npm-debug.log* +yarn-debug.log* +yarn-error.log* +.pnpm-debug.log* + +# env files (can opt-in for committing if needed) +.env* + +# vercel +.vercel + +# typescript +*.tsbuildinfo +next-env.d.ts + diff --git a/examples/computer-use-nextjs/README.md b/examples/computer-use-nextjs/README.md new file mode 100644 index 0000000..e215bc4 --- /dev/null +++ b/examples/computer-use-nextjs/README.md @@ -0,0 +1,36 @@ +This is a [Next.js](https://nextjs.org) project bootstrapped with [`create-next-app`](https://nextjs.org/docs/app/api-reference/cli/create-next-app). + +## Getting Started + +First, run the development server: + +```bash +npm run dev +# or +yarn dev +# or +pnpm dev +# or +bun dev +``` + +Open [http://localhost:3000](http://localhost:3000) with your browser to see the result. + +You can start editing the page by modifying `app/page.tsx`. The page auto-updates as you edit the file. + +This project uses [`next/font`](https://nextjs.org/docs/app/building-your-application/optimizing/fonts) to automatically optimize and load [Geist](https://vercel.com/font), a new font family for Vercel. + +## Learn More + +To learn more about Next.js, take a look at the following resources: + +- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API. +- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial. + +You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js) - your feedback and contributions are welcome! + +## Deploy on Vercel + +The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js. + +Check out our [Next.js deployment documentation](https://nextjs.org/docs/app/building-your-application/deploying) for more details. diff --git a/examples/computer-use-nextjs/app/api/chat/route.ts b/examples/computer-use-nextjs/app/api/chat/route.ts new file mode 100644 index 0000000..cd9a852 --- /dev/null +++ b/examples/computer-use-nextjs/app/api/chat/route.ts @@ -0,0 +1,67 @@ +import { anthropic } from "@ai-sdk/anthropic"; +import { streamText, UIMessage } from "ai"; +import { killBrowser } from "@/lib/kernel/utils"; +import { createComputerTool } from "@/lib/kernel/tool"; +import { prunedMessages } from "@/lib/utils"; + +// Allow streaming responses up to 5 minutes +export const maxDuration = 300; + +export async function POST(req: Request) { + const { + messages, + sandboxId, + }: { + messages: UIMessage[]; + sandboxId: string; + } = await req.json(); + + try { + // Log request to monitor token usage + console.log( + `[${new Date().toISOString()}] Chat request - Messages: ${ + messages.length + }, SandboxId: ${sandboxId}` + ); + + const result = streamText({ + model: anthropic("claude-3-7-sonnet-20250219"), + system: + "You are an autonomous agent with access to a computer. " + + "Use the computer tool to help the user with their requests. " + + "Do not provide conversational responses - just use the tools to complete the task. " + + "Work silently and efficiently. Only use tools to perform actions. " + + "If the browser opens with a setup wizard, YOU MUST IGNORE IT and move straight to the next step (e.g. input the url in the search bar).", + messages: prunedMessages(messages), + tools: { + computer: createComputerTool(sandboxId), + }, + providerOptions: { + anthropic: { cacheControl: { type: "ephemeral" } }, + }, + }); + + // Create response stream + const response = result.toDataStreamResponse({ + // @ts-expect-error - AI SDK type mismatch + getErrorMessage(error) { + console.error("Stream error:", error); + + return error; + }, + }); + + return response; + } catch (error) { + console.error("Chat API error:", error); + + // Force cleanup on error + console.log(`killing browser with id: ${sandboxId}`); + await killBrowser(sandboxId); + + return new Response(JSON.stringify({ error: "Internal Server Error" }), { + status: 500, + headers: { "Content-Type": "application/json" }, + }); + } +} diff --git a/examples/computer-use-nextjs/app/api/kill-desktop/route.ts b/examples/computer-use-nextjs/app/api/kill-desktop/route.ts new file mode 100644 index 0000000..f2ea028 --- /dev/null +++ b/examples/computer-use-nextjs/app/api/kill-desktop/route.ts @@ -0,0 +1,28 @@ +import { killBrowser } from "@/lib/kernel/utils"; +import { NextRequest } from "next/server"; + +export async function POST(req: NextRequest) { + try { + const sandboxId = req.nextUrl.searchParams.get("sandboxId"); + + if (!sandboxId) { + return new Response( + JSON.stringify({ error: "sandboxId is required" }), + { status: 400 } + ); + } + + await killBrowser(sandboxId); + + return new Response( + JSON.stringify({ success: true }), + { status: 200 } + ); + } catch (error) { + console.error("Failed to kill browser:", error); + return new Response( + JSON.stringify({ error: "Failed to kill browser" }), + { status: 500 } + ); + } +} \ No newline at end of file diff --git a/examples/computer-use-nextjs/app/favicon.ico b/examples/computer-use-nextjs/app/favicon.ico new file mode 100644 index 0000000..718d6fe Binary files /dev/null and b/examples/computer-use-nextjs/app/favicon.ico differ diff --git a/examples/computer-use-nextjs/app/globals.css b/examples/computer-use-nextjs/app/globals.css new file mode 100644 index 0000000..a2dc41e --- /dev/null +++ b/examples/computer-use-nextjs/app/globals.css @@ -0,0 +1,26 @@ +@import "tailwindcss"; + +:root { + --background: #ffffff; + --foreground: #171717; +} + +@theme inline { + --color-background: var(--background); + --color-foreground: var(--foreground); + --font-sans: var(--font-geist-sans); + --font-mono: var(--font-geist-mono); +} + +@media (prefers-color-scheme: dark) { + :root { + --background: #0a0a0a; + --foreground: #ededed; + } +} + +body { + background: var(--background); + color: var(--foreground); + font-family: Arial, Helvetica, sans-serif; +} diff --git a/examples/computer-use-nextjs/app/layout.tsx b/examples/computer-use-nextjs/app/layout.tsx new file mode 100644 index 0000000..f7fa87e --- /dev/null +++ b/examples/computer-use-nextjs/app/layout.tsx @@ -0,0 +1,34 @@ +import type { Metadata } from "next"; +import { Geist, Geist_Mono } from "next/font/google"; +import "./globals.css"; + +const geistSans = Geist({ + variable: "--font-geist-sans", + subsets: ["latin"], +}); + +const geistMono = Geist_Mono({ + variable: "--font-geist-mono", + subsets: ["latin"], +}); + +export const metadata: Metadata = { + title: "Create Next App", + description: "Generated by create next app", +}; + +export default function RootLayout({ + children, +}: Readonly<{ + children: React.ReactNode; +}>) { + return ( + + + {children} + + + ); +} diff --git a/examples/computer-use-nextjs/app/page.tsx b/examples/computer-use-nextjs/app/page.tsx new file mode 100644 index 0000000..79fb79a --- /dev/null +++ b/examples/computer-use-nextjs/app/page.tsx @@ -0,0 +1,326 @@ +"use client"; + +import { PreviewMessage } from "@/components/message"; +import { getBrowserURL } from "@/lib/kernel/utils"; +import { useScrollToBottom } from "@/lib/use-scroll-to-bottom"; +import { useChat } from "@ai-sdk/react"; +import { useEffect, useState } from "react"; +import { Input } from "@/components/input"; +import { Button } from "@/components/ui/button"; +import { toast } from "sonner"; +import { DeployButton, ProjectInfo } from "@/components/project-info"; +import { AISDKLogo } from "@/components/icons"; +import { PromptSuggestions } from "@/components/prompt-suggestions"; +import { + ResizableHandle, + ResizablePanel, + ResizablePanelGroup, +} from "@/components/ui/resizable"; +import { ABORTED } from "@/lib/utils"; + +export default function Chat() { + // Create separate refs for mobile and desktop to ensure both scroll properly + const [desktopContainerRef, desktopEndRef] = useScrollToBottom(); + const [mobileContainerRef, mobileEndRef] = useScrollToBottom(); + + const [isInitializing, setIsInitializing] = useState(true); + const [streamUrl, setStreamUrl] = useState(null); + const [sandboxId, setSandboxId] = useState(null); + + const { + messages, + input, + handleInputChange, + handleSubmit, + status, + stop: stopGeneration, + append, + setMessages, + } = useChat({ + api: "/api/chat", + id: sandboxId ?? undefined, + body: { + sandboxId, + }, + maxSteps: 30, + onError: (error) => { + console.error(error); + + // Check if it's a rate limit error + const errorMessage = error?.message || error?.toString() || ""; + const isRateLimit = + errorMessage.includes("rate limit") || + errorMessage.includes("Rate Limit") || + errorMessage.includes("429"); + + if (isRateLimit) { + toast.error("Rate limit exceeded", { + description: "You've hit the API rate limit. Please wait a moment and try again.", + richColors: true, + position: "top-center", + duration: 5000, + }); + } else { + toast.error("There was an error", { + description: "Please try again later.", + richColors: true, + position: "top-center", + }); + } + }, + }); + + const stop = () => { + stopGeneration(); + + const lastMessage = messages.at(-1); + const lastMessageLastPart = lastMessage?.parts.at(-1); + if ( + lastMessage?.role === "assistant" && + lastMessageLastPart?.type === "tool-invocation" + ) { + setMessages((prev) => [ + ...prev.slice(0, -1), + { + ...lastMessage, + parts: [ + ...lastMessage.parts.slice(0, -1), + { + ...lastMessageLastPart, + toolInvocation: { + ...lastMessageLastPart.toolInvocation, + state: "result", + result: ABORTED, + }, + }, + ], + }, + ]); + } + }; + + const isLoading = status !== "ready"; + + const refreshBrowser = async () => { + try { + setIsInitializing(true); + const { streamUrl, id } = await getBrowserURL(sandboxId || undefined); + setStreamUrl(streamUrl); + setSandboxId(id); + } catch (err) { + console.error("Failed to refresh browser:", err); + } finally { + setIsInitializing(false); + } + }; + + // Kill browser on page close + useEffect(() => { + if (!sandboxId) return; + + // Function to kill the browser - just one method to reduce duplicates + const killBrowser = () => { + if (!sandboxId) return; + + // Use sendBeacon which is best supported across browsers + navigator.sendBeacon( + `/api/kill-desktop?sandboxId=${encodeURIComponent(sandboxId)}`, + ); + }; + + // Detect iOS / Safari + const isIOS = + /iPad|iPhone|iPod/.test(navigator.userAgent) || + (navigator.platform === "MacIntel" && navigator.maxTouchPoints > 1); + const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); + + // Choose exactly ONE event handler based on the browser + if (isIOS || isSafari) { + // For Safari on iOS, use pagehide which is most reliable + window.addEventListener("pagehide", killBrowser); + + return () => { + window.removeEventListener("pagehide", killBrowser); + // Also kill browser when component unmounts + killBrowser(); + }; + } else { + // For all other browsers, use beforeunload + window.addEventListener("beforeunload", killBrowser); + + return () => { + window.removeEventListener("beforeunload", killBrowser); + // Also kill browser when component unmounts + killBrowser(); + }; + } + }, [sandboxId]); + + useEffect(() => { + // Initialize browser and get stream URL when the component mounts + const init = async () => { + try { + setIsInitializing(true); + + // Use the provided ID or create a new one + const { streamUrl, id } = await getBrowserURL(sandboxId ?? undefined); + + setStreamUrl(streamUrl); + setSandboxId(id); + } catch (err) { + console.error("Failed to initialize:", err); + toast.error("Failed to initialize browser"); + } finally { + setIsInitializing(false); + } + }; + + init(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + return ( +
+ {/* Mobile/tablet banner */} +
+ Headless mode +
+ + {/* Resizable Panels */} +
+ + {/* Desktop Stream Panel */} + + {streamUrl ? ( + <> +