Implement Anthropic Computer Use as a sample app (#9)

* Replace xdotool and gnome-screenshot with playwright

Author: juecd

README.md

@@ -38,6 +38,8 @@ create-kernel-app [app-name] [options]
   - `sample-app`: Basic template with Playwright integration
   - `browser-use`: Template with Browser Use SDK (Python only)
   - `stagehand`: Template with Stagehand SDK (Typescript only)
+  - `persistent-browser`: Implements `sample-app` using a persistent browser 
+  - `computer-use`: Implements a prompt loop using Anthropic Computer Use
 
 ### Examples
 
@@ -51,6 +53,11 @@ Create a Typescript application with Stagehand template:
 npx @onkernel/create-kernel-app my-app --language typescript --template stagehand
 ```
 
+Create a Typescript application with Computer Use template:
+```bash
+npx @onkernel/create-kernel-app my-app --language typescript --template computer-use
+```
+
 Create a Python application with a sample app:
 ```bash
 npx @onkernel/create-kernel-app my-app --language python --template sample-app
@@ -60,6 +67,7 @@ Create a Python application with Browser Use template:
 ```bash
 npx @onkernel/create-kernel-app my-app --language python --template browser-use
 ```
+```
 
 ## Next Steps
 
@@ -82,7 +90,7 @@ export KERNEL_API_KEY=<YOUR_API_KEY>
 4. Deploy your application:
 ```bash
 # Typscript
-kernel deploy index.ts  # --env OPENAI_API_KEY=XXX if Stagehand
+kernel deploy index.ts  # --env OPENAI_API_KEY=XXX if Stagehand; --env ANTHROPIC_API_KEY=XXX if Computer Use
 
 # Python
 kernel deploy main.py   # --env OPENAI_API_KEY=XXX if Browser Use
@@ -98,6 +106,9 @@ kernel invoke ts-basic get-page-title --payload '{"url": "https://www.google.com
 # Typescript + Stagehand
 kernel invoke ts-stagehand stagehand-task --payload '{"query": "Best wired earbuds"}'
 
+# Typescript + Computer Use
+kernel invoke ts-cu cu-task --payload '{"query": "Search for the top 3 restaurants in NYC according to Pete Wells"}'
+
 # Python + Sample App
 kernel invoke python-basic get-page-title --payload '{"url": "https://www.google.com"}'
 
@@ -114,6 +125,8 @@ These are the sample apps currently available when you run `npx @onkernel/create
 | **sample-app** | Returns the page title of a specified URL | Playwright | `{ url }` |
 | **browser-use** | Completes a specified task | Browser Use | `{ task }` |
 | **stagehand** | Returns the first result of a specified Google search | Stagehand | `{ query }` |
+| **persistent-browser** | Implements `sample-app` using a persistent browser | Playwright | `{ url }` |
+| **computer-use** | Implements a prompt loop | Anthropic Computer Use API | `{ query }` |
 
 ## Documentation
 

index.ts

@@ -17,7 +17,8 @@ type TemplateKey =
   | "sample-app"
   | "browser-use"
   | "stagehand"
-  | "persistent-browser";
+  | "persistent-browser"
+  | "computer-use";
 type LanguageInfo = { name: string; shorthand: string };
 type TemplateInfo = {
   name: string;
@@ -32,6 +33,7 @@ const TEMPLATE_SAMPLE_APP = "sample-app";
 const TEMPLATE_BROWSER_USE = "browser-use";
 const TEMPLATE_STAGEHAND = "stagehand";
 const TEMPLATE_PERSISTENT_BROWSER = "persistent-browser";
+const TEMPLATE_COMPUTER_USE = "computer-use";
 const LANGUAGE_SHORTHAND_TS = "ts";
 const LANGUAGE_SHORTHAND_PY = "py";
 
@@ -66,6 +68,11 @@ const TEMPLATES: Record<TemplateKey, TemplateInfo> = {
       "Implements a persistent browser that maintains state across invocations",
     languages: [LANGUAGE_TYPESCRIPT],
   },
+  [TEMPLATE_COMPUTER_USE]: {
+    name: "Computer Use",
+    description: "Implements the Anthropic Computer Use SDK",
+    languages: [LANGUAGE_TYPESCRIPT],
+  },
 };
 
 const INVOKE_SAMPLES: Record<
@@ -79,6 +86,8 @@ const INVOKE_SAMPLES: Record<
       'kernel invoke ts-stagehand stagehand-task --payload \'{"query": "Best wired earbuds"}\'',
     [TEMPLATE_PERSISTENT_BROWSER]:
       'kernel invoke ts-persistent-browser persistent-browser-task --payload \'{"url": "https://news.ycombinator.com/"}\'',
+    [TEMPLATE_COMPUTER_USE]:
+      'kernel invoke ts-cu cu-task --payload \'{"query": "Return the first url of a search result for NYC restaurant reviews Pete Wells"}\'',
   },
   [LANGUAGE_PYTHON]: {
     [TEMPLATE_SAMPLE_APP]:
@@ -299,10 +308,12 @@ function printNextSteps(
 ): void {
   // Determine which sample command to show based on language and template
   const deployCommand =
-    language === LANGUAGE_TYPESCRIPT && template === TEMPLATE_SAMPLE_APP
+    language === LANGUAGE_TYPESCRIPT && (template === TEMPLATE_SAMPLE_APP || template === TEMPLATE_PERSISTENT_BROWSER)
       ? "kernel deploy index.ts"
       : language === LANGUAGE_TYPESCRIPT && template === TEMPLATE_STAGEHAND
       ? "kernel deploy index.ts --env OPENAI_API_KEY=XXX"
+      : language === LANGUAGE_TYPESCRIPT && template === TEMPLATE_COMPUTER_USE
+      ? "kernel deploy index.ts --env ANTHROPIC_API_KEY=XXX"
       : language === LANGUAGE_PYTHON && template === TEMPLATE_SAMPLE_APP
       ? "kernel deploy main.py"
       : language === LANGUAGE_PYTHON && template === TEMPLATE_BROWSER_USE
@@ -341,7 +352,7 @@ program
   )
   .option(
     "-t, --template <template>",
-    `Template type (${TEMPLATE_SAMPLE_APP}, ${TEMPLATE_BROWSER_USE}, ${TEMPLATE_STAGEHAND})`
+    `Template type (${TEMPLATE_SAMPLE_APP}, ${TEMPLATE_BROWSER_USE}, ${TEMPLATE_STAGEHAND}, ${TEMPLATE_PERSISTENT_BROWSER}, ${TEMPLATE_COMPUTER_USE})`
   )
   .action(
     async (

templates/typescript/computer-use/.gitignore

@@ -0,0 +1,39 @@
+# Dependencies
+node_modules/
+package-lock.json
+
+# TypeScript
+*.tsbuildinfo
+dist/
+build/
+
+# Environment
+.env
+.env.local
+.env.*.local
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Logs
+logs/
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Testing
+coverage/
+.nyc_output/
+
+# Misc
+.cache/
+.temp/
+.tmp/ 

templates/typescript/computer-use/README.md

@@ -0,0 +1,7 @@
+# Kernel Typscript Sample App - Computer Use
+
+This is a simple Kernel application that implements a prompt loop using Anthropic Computer Use.
+
+It generally follows the [Anthropic Reference Implementation](https://github.com/anthropics/anthropic-quickstarts/tree/main/computer-use-demo) but replaces `xodotool` and `gnome-screenshot` with Playwright.
+
+See the [docs](https://docs.onkernel.com/quickstart) for information.

templates/typescript/computer-use/index.ts

@@ -0,0 +1,82 @@
+import { Kernel, type KernelContext } from '@onkernel/sdk';
+import { samplingLoop } from './loop';
+import { chromium } from 'playwright';
+
+const kernel = new Kernel();
+
+const app = kernel.app('ts-cu');
+
+interface QueryInput {
+  query: string;
+}
+
+interface QueryOutput {
+  result: string;
+}
+
+// LLM API Keys are set in the environment during `kernel deploy <filename> -e ANTHROPIC_API_KEY=XXX`
+// See https://docs.onkernel.com/launch/deploy#environment-variables
+const ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY;
+
+if (!ANTHROPIC_API_KEY) {
+  throw new Error('ANTHROPIC_API_KEY is not set');
+}
+
+app.action<QueryInput, QueryOutput>(
+  'cu-task',
+  async (ctx: KernelContext, payload?: QueryInput): Promise<QueryOutput> => {
+    if (!payload?.query) {
+      throw new Error('Query is required');
+    }
+
+    const kernelBrowser = await kernel.browsers.create({
+      invocation_id: ctx.invocation_id,
+    });
+
+    console.log("Kernel browser live view url: ", kernelBrowser.browser_live_view_url);
+
+    const browser = await chromium.connectOverCDP(kernelBrowser.cdp_ws_url);
+    const context = await browser.contexts()[0];
+    const page = await context?.pages()[0];
+    if (!page) {
+      throw new Error('Error getting initial page');
+    }
+
+    try {
+      // Run the sampling loop
+      const finalMessages = await samplingLoop({
+        model: 'claude-sonnet-4-20250514',
+        messages: [{
+          role: 'user',
+          content: payload.query,
+        }],
+        apiKey: ANTHROPIC_API_KEY,
+        thinkingBudget: 1024,
+        playwrightPage: page,
+      });
+
+      // Extract the final result from the messages
+      if (finalMessages.length === 0) {
+        throw new Error('No messages were generated during the sampling loop');
+      }
+
+      const lastMessage = finalMessages[finalMessages.length - 1];
+      if (!lastMessage) {
+        throw new Error('Failed to get the last message from the sampling loop');
+      }
+
+      const result = typeof lastMessage.content === 'string' 
+        ? lastMessage.content 
+        : lastMessage.content.map(block => 
+            block.type === 'text' ? block.text : ''
+          ).join('');
+
+      return { result };
+    } catch (error) {
+      console.error('Error in sampling loop:', error);
+      throw error;
+    } finally {
+      await browser.close();
+    }
+  },
+);