feat(cli): add TypeScript runner with AgentProxy for browser automation

Add a new ts-runner module that enables running TypeScript automation scripts
directly from the CLI. The AgentProxy class provides a simplified interface
for browser automation with CDP connection or browser launch support.

Author: quanru

packages/cli/bin/midscene

@@ -1,3 +1,29 @@
 #!/usr/bin/env node
 
-require('../dist/lib/index.js')
+const path = require('node:path');
+const { spawn } = require('node:child_process');
+
+const args = process.argv.slice(2);
+const scriptArg = args.find((arg) => !arg.startsWith('-'));
+
+const isTypeScriptFile =
+  scriptArg && (scriptArg.endsWith('.ts') || scriptArg.endsWith('.mts'));
+
+if (isTypeScriptFile) {
+  const runnerPath = path.join(__dirname, '../dist/es/ts-runner/runner.mjs');
+  const tsxPath = require.resolve('tsx/cli');
+
+  const child = spawn(
+    process.execPath,
+    [tsxPath, runnerPath, ...args],
+    { stdio: 'inherit', cwd: process.cwd() },
+  );
+
+  child.on('exit', (code) => process.exit(code || 0));
+  child.on('error', (error) => {
+    console.error('Failed to start tsx:', error.message);
+    process.exit(1);
+  });
+} else {
+  require('../dist/lib/index.js');
+}

packages/cli/examples/ai-todo-declarative.mts

@@ -0,0 +1,48 @@
+/// <reference path="../src/ts-runner/global.d.ts" />
+
+// TodoMVC automation test script (declarative style)
+// Usage: midscene examples/ai-todo-declarative.mts
+
+import type { AgentProxy } from '../src/ts-runner/agent-proxy';
+
+export const launch = {
+  headed: true,
+  url: 'https://todomvc.com/examples/react/dist/',
+};
+
+export async function run(agent: AgentProxy) {
+  console.log('Starting TodoMVC test (declarative style)...');
+
+  // Create a task
+  await agent.aiAct(
+    'Enter "Learn TypeScript" in the task box, then press Enter to create',
+  );
+
+  // Query all tasks
+  const tasks = await agent.aiQuery<string[]>('string[], tasks in the list');
+  console.log('Tasks:', tasks);
+
+  // Verify task created
+  if (!tasks.includes('Learn TypeScript')) {
+    throw new Error('Task "Learn TypeScript" not found');
+  }
+  console.log('Task created successfully');
+
+  // Complete the task
+  await agent.aiAct('Click the checkbox next to the first task');
+  console.log('Task completed');
+
+  // Query completed tasks
+  await agent.aiAct('Click the "Completed" status button below the task list');
+  const completedTasks = await agent.aiQuery<string[]>(
+    'string[], Extract all task names from the list',
+  );
+  console.log('Completed tasks:', completedTasks);
+
+  if (completedTasks.length !== 1) {
+    throw new Error(`Expected 1 completed task, got ${completedTasks.length}`);
+  }
+  console.log('Completed tasks verified');
+
+  console.log('\nAll tests passed!');
+}

packages/cli/examples/ai-todo.mts

@@ -0,0 +1,85 @@
+/// <reference path="../src/ts-runner/global.d.ts" />
+
+// TodoMVC automation test script
+// Usage: midscene examples/ai-todo.mts
+
+// Launch browser and open TodoMVC
+await agent.launch({
+  headed: true,
+  url: 'https://todomvc.com/examples/react/dist/',
+});
+
+console.log('Starting TodoMVC test...');
+
+// Create tasks
+await agent.aiAct(
+  'Enter "Learn JS today" in the task box, then press Enter to create',
+);
+await agent.aiAct(
+  'Enter "Learn Rust tomorrow" in the task box, then press Enter to create',
+);
+await agent.aiAct(
+  'Enter "Learning AI the day after tomorrow" in the task box, then press Enter to create',
+);
+
+// Query all tasks
+const allTaskList = await agent.aiQuery<string[]>(
+  'string[], tasks in the list',
+);
+console.log('All tasks:', allTaskList);
+
+// Verify tasks created successfully
+const expectedTasks = [
+  'Learn JS today',
+  'Learn Rust tomorrow',
+  'Learning AI the day after tomorrow',
+];
+for (const task of expectedTasks) {
+  if (!allTaskList.includes(task)) {
+    throw new Error(`Task "${task}" not found`);
+  }
+}
+console.log('All tasks created successfully');
+
+// Delete the second task
+await agent.aiAct('Move your mouse over the second item in the task list');
+await agent.aiAct('Click the delete button to the right of the second task');
+console.log('Second task deleted');
+
+// Complete the second task (now the original third one)
+await agent.aiAct('Click the checkbox next to the second task');
+console.log('Second task completed');
+
+// View completed tasks
+await agent.aiAct('Click the "Completed" status button below the task list');
+
+// Query completed tasks list
+const completedTasks = await agent.aiQuery<string[]>(
+  'string[], Extract all task names from the list',
+);
+console.log('Completed tasks:', completedTasks);
+
+if (completedTasks.length !== 1) {
+  throw new Error(`Expected 1 completed task, got ${completedTasks.length}`);
+}
+if (completedTasks[0] !== 'Learning AI the day after tomorrow') {
+  throw new Error(
+    `Expected task name "Learning AI the day after tomorrow", got "${completedTasks[0]}"`,
+  );
+}
+console.log('Completed tasks verified');
+
+// Query input placeholder
+const placeholder = await agent.aiQuery<string>(
+  'string, return the placeholder text in the input box',
+);
+console.log('Input placeholder:', placeholder);
+
+if (placeholder !== 'What needs to be done?') {
+  throw new Error(
+    `Expected placeholder "What needs to be done?", got "${placeholder}"`,
+  );
+}
+console.log('Placeholder verified');
+
+console.log('\nAll tests passed!');

packages/cli/package.json

@@ -6,10 +6,26 @@
   "homepage": "https://midscenejs.com/",
   "main": "./dist/lib/index.js",
   "module": "./dist/es/index.mjs",
+  "types": "./dist/types/index.d.ts",
   "bin": {
     "midscene": "./bin/midscene"
   },
-  "files": ["dist", "README.md", "bin"],
+  "exports": {
+    ".": {
+      "import": "./dist/es/index.mjs",
+      "require": "./dist/lib/index.js",
+      "types": "./dist/types/index.d.ts"
+    },
+    "./global": {
+      "types": "./src/ts-runner/global.d.ts"
+    }
+  },
+  "typesVersions": {
+    "*": {
+      "global": ["./src/ts-runner/global.d.ts"]
+    }
+  },
+  "files": ["dist", "README.md", "bin", "src/ts-runner/global.d.ts"],
   "scripts": {
     "dev": "npm run build:watch",
     "build": "rslib build",
@@ -24,9 +40,11 @@
     "@midscene/ios": "workspace:*",
     "@midscene/shared": "workspace:*",
     "@midscene/web": "workspace:*",
+    "dotenv": "^16.4.5",
     "http-server": "14.1.1",
     "lodash.merge": "4.6.2",
-    "puppeteer": "24.6.0"
+    "puppeteer": "24.6.0",
+    "tsx": "^4.19.2"
   },
   "devDependencies": {
     "@rslib/core": "^0.18.3",
@@ -37,7 +55,6 @@
     "@types/yargs": "17.0.32",
     "chalk": "4.1.2",
     "cli-spinners": "3.2.0",
-    "dotenv": "^16.4.5",
     "execa": "9.3.0",
     "glob": "11.0.0",
     "js-yaml": "4.1.0",

packages/cli/rslib.config.ts

@@ -41,6 +41,8 @@ export default defineConfig({
   source: {
     entry: {
       index: 'src/index.ts',
+      'ts-runner/runner': 'src/ts-runner/runner.ts',
+      'ts-runner/index': 'src/ts-runner/index.ts',
     },
     define: {
       __VERSION__: JSON.stringify(version),

packages/cli/src/ts-runner/agent-proxy.ts

@@ -0,0 +1,166 @@
+import type { Browser, Page } from 'puppeteer';
+import type { CdpConfig, LaunchConfig } from './types';
+
+export class AgentProxy {
+  private browser: Browser | null = null;
+  private page: Page | null = null;
+  private innerAgent: any = null;
+  private isOwned = false;
+
+  async connect(config?: CdpConfig): Promise<void> {
+    if (!config) {
+      const endpoint = await this.discoverLocal();
+      await this.connectToEndpoint(endpoint);
+      return;
+    }
+
+    if (typeof config === 'string') {
+      await this.connectToEndpoint(config);
+      return;
+    }
+
+    let endpoint = config.endpoint;
+    if (config.apiKey) {
+      const url = new URL(endpoint);
+      url.searchParams.set('apiKey', config.apiKey);
+      endpoint = url.toString();
+    }
+
+    await this.connectToEndpoint(endpoint);
+
+    if (config.tabUrl || typeof config.tabIndex === 'number') {
+      await this.selectTab(config);
+    }
+  }
+
+  async launch(config: LaunchConfig = {}): Promise<void> {
+    const puppeteer = await import('puppeteer');
+
+    this.browser = await puppeteer.default.launch({
+      headless: !config.headed,
+    });
+    this.isOwned = true;
+
+    this.page = await this.browser.newPage();
+
+    if (config.viewport) {
+      await this.page.setViewport(config.viewport);
+    }
+
+    if (config.url) {
+      await this.page.goto(config.url, { waitUntil: 'domcontentloaded' });
+    }
+
+    await this.createAgent();
+  }
+
+  async aiAct(prompt: string, options?: any): Promise<any> {
+    this.ensureConnected();
+    return this.innerAgent.aiAct(prompt, options);
+  }
+
+  async aiAction(prompt: string, options?: any): Promise<any> {
+    this.ensureConnected();
+    return this.innerAgent.aiAction(prompt, options);
+  }
+
+  async aiQuery<T = any>(prompt: string, options?: any): Promise<T> {
+    this.ensureConnected();
+    return this.innerAgent.aiQuery(prompt, options);
+  }
+
+  async aiAssert(assertion: string, options?: any): Promise<void> {
+    this.ensureConnected();
+    return this.innerAgent.aiAssert(assertion, options);
+  }
+
+  async aiLocate(prompt: string, options?: any): Promise<any> {
+    this.ensureConnected();
+    return this.innerAgent.aiLocate(prompt, options);
+  }
+
+  async aiWaitFor(assertion: string, options?: any): Promise<void> {
+    this.ensureConnected();
+    return this.innerAgent.aiWaitFor(assertion, options);
+  }
+
+  async destroy(): Promise<void> {
+    if (this.innerAgent) {
+      await this.innerAgent.destroy();
+      this.innerAgent = null;
+    }
+
+    if (this.browser) {
+      if (this.isOwned) {
+        await this.browser.close();
+      } else {
+        this.browser.disconnect();
+      }
+      this.browser = null;
+    }
+
+    this.page = null;
+  }
+
+  private async discoverLocal(port = 9222): Promise<string> {
+    const response = await fetch(`http://localhost:${port}/json/version`);
+    if (!response.ok) {
+      throw new Error(
+        `Cannot connect to local Chrome (port ${port}).
+Please start Chrome with the following command:
+  macOS: open -a "Google Chrome" --args --remote-debugging-port=${port}
+  Linux: google-chrome --remote-debugging-port=${port}
+  Windows: chrome.exe --remote-debugging-port=${port}`,
+      );
+    }
+    const info = (await response.json()) as { webSocketDebuggerUrl: string };
+    return info.webSocketDebuggerUrl;
+  }
+
+  private async connectToEndpoint(endpoint: string): Promise<void> {
+    const puppeteer = await import('puppeteer');
+
+    this.browser = await puppeteer.default.connect({
+      browserWSEndpoint: endpoint,
+    });
+    this.isOwned = false;
+
+    const pages = await this.browser.pages();
+    this.page = pages[0] || (await this.browser.newPage());
+    await this.createAgent();
+  }
+
+  private async selectTab(config: {
+    tabUrl?: string;
+    tabIndex?: number;
+  }): Promise<void> {
+    if (!this.browser) return;
+
+    const pages = await this.browser.pages();
+    let targetPage: Page | undefined;
+
+    if (config.tabUrl) {
+      targetPage = pages.find((p) => p.url().includes(config.tabUrl!));
+    } else if (typeof config.tabIndex === 'number') {
+      targetPage = pages[config.tabIndex];
+    }
+
+    if (targetPage) {
+      this.page = targetPage;
+      await this.createAgent();
+    }
+  }
+
+  private async createAgent(): Promise<void> {
+    const { PuppeteerAgent } = await import('@midscene/web/puppeteer');
+    this.innerAgent = new PuppeteerAgent(this.page!);
+  }
+
+  private ensureConnected(): void {
+    if (!this.innerAgent) {
+      throw new Error(
+        'Please call agent.connect() or agent.launch() first to connect to a browser',
+      );
+    }
+  }
+}