Better browser experience: show browser, take over user session, content filter, robustness improvements

packages/agent/CHANGELOG.md

@@ -1,5 +1,11 @@
 # mycoder-agent
 
+## 0.3.0
+
+### Minor Changes
+
+- Better browser experience: show browser, take over user session, content filter, robustness improvements to browsing.
+
 ## 0.2.1
 
 ### Patch Changes

packages/agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mycoder-agent",
-  "version": "0.2.1",
+  "version": "0.3.0",
   "description": "Agent module for mycoder - an AI-powered software development assistant",
   "type": "module",
   "main": "dist/index.js",
@@ -45,10 +45,12 @@
   "license": "MIT",
   "dependencies": {
     "@anthropic-ai/sdk": "^0.37",
+    "@mozilla/readability": "^0.5.0",
     "@playwright/test": "^1.50.1",
     "@vitest/browser": "^3.0.5",
     "chalk": "^5",
     "dotenv": "^16",
+    "jsdom": "^26.0.0",
     "playwright": "^1.50.1",
     "uuid": "^11",
     "zod": "^3",

packages/agent/src/core/executeToolCall.ts

@@ -23,7 +23,7 @@ export const executeToolCall = async (
     customPrefix: tool.logPrefix,
   });
 
-  const toolContext = {
+  const toolContext: ToolContext = {
     ...context,
     logger,
   };

packages/agent/src/core/toolAgent.respawn.test.ts

@@ -5,11 +5,14 @@ import { getTools } from '../../src/tools/getTools.js';
 import { MockLogger } from '../utils/mockLogger.js';
 
 import { TokenTracker } from './tokens.js';
+import { ToolContext } from './types.js';
 
-const toolContext = {
+const toolContext: ToolContext = {
   logger: new MockLogger(),
   headless: true,
   workingDirectory: '.',
+  userSession: false,
+  pageFilter: 'simple',
   tokenTracker: new TokenTracker(),
 };
 // Mock Anthropic SDK

packages/agent/src/core/toolAgent.test.ts

@@ -5,12 +5,14 @@ import { MockLogger } from '../utils/mockLogger.js';
 import { executeToolCall } from './executeToolCall.js';
 import { TokenTracker } from './tokens.js';
 import { toolAgent } from './toolAgent.js';
-import { Tool } from './types.js';
+import { Tool, ToolContext } from './types.js';
 
-const toolContext = {
+const toolContext: ToolContext = {
   logger: new MockLogger(),
   headless: true,
   workingDirectory: '.',
+  userSession: false,
+  pageFilter: 'simple',
   tokenTracker: new TokenTracker(),
 };
 

packages/agent/src/core/types.ts

@@ -6,14 +6,18 @@
 
 export type TokenLevel = 'debug' | 'verbose' | 'info' | 'warn' | 'error';
 
+export type pageFilter = 'simple' | 'none' | 'readability';
+
 export type ToolContext = {
   logger: Logger;
   workingDirectory: string;
   headless: boolean;
+  userSession: boolean;
+  pageFilter: pageFilter;
   tokenTracker: TokenTracker;
 };
 
 export type Tool<TParams = Record<string, any>, TReturn = any> = {
   name: string;
   description: string;
   parameters: JsonSchema7Type;
@@ -29,7 +33,7 @@
 export type ToolCall = {
   id: string;
   name: string;
   input: any;
 };
 
 export type TextContent = {

packages/agent/src/tools/browser/browseMessage.ts

@@ -3,7 +3,9 @@ import { zodToJsonSchema } from 'zod-to-json-schema';
 
 import { Tool } from '../../core/types.js';
 import { errorToString } from '../../utils/errorToString.js';
+import { sleep } from '../../utils/sleep.js';
 
+import { filterPageContent } from './filterPageContent.js';
 import { browserSessions, type BrowserAction, SelectorType } from './types.js';
 
 // Schema for browser action
@@ -71,8 +73,29 @@ export const browseMessageTool: Tool<Parameters, ReturnType> = {
   parameters: zodToJsonSchema(parameterSchema),
   returns: zodToJsonSchema(returnSchema),
 
-  execute: async ({ instanceId, action }, { logger }): Promise<ReturnType> => {
+  execute: async (
+    { instanceId, action },
+    { logger, pageFilter },
+  ): Promise<ReturnType> => {
+    // Validate action format
+    if (!action || typeof action !== 'object') {
+      logger.error('Invalid action format: action must be an object');
+      return {
+        status: 'error',
+        error: 'Invalid action format: action must be an object',
+      };
+    }
+
+    if (!action.actionType) {
+      logger.error('Invalid action format: actionType is required');
+      return {
+        status: 'error',
+        error: 'Invalid action format: actionType is required',
+      };
+    }
+
     logger.verbose(`Executing browser action: ${action.actionType}`);
+    logger.verbose(`Webpage processing mode: ${pageFilter}`);
 
     try {
       const session = browserSessions.get(instanceId);
@@ -87,10 +110,44 @@ export const browseMessageTool: Tool<Parameters, ReturnType> = {
           if (!action.url) {
             throw new Error('URL required for goto action');
           }
-          await page.goto(action.url, { waitUntil: 'networkidle' });
-          const content = await page.content();
-          logger.verbose('Navigation completed successfully');
-          return { status: 'success', content };
+
+          try {
+            // Try with 'domcontentloaded' first which is more reliable than 'networkidle'
+            logger.verbose(
+              `Navigating to ${action.url} with 'domcontentloaded' waitUntil`,
+            );
+            await page.goto(action.url, { waitUntil: 'domcontentloaded' });
+            await sleep(3000);
+            const content = await filterPageContent(page, pageFilter);
+            logger.verbose(`Content: ${content}`);
+            logger.verbose(
+              'Navigation completed with domcontentloaded strategy',
+            );
+            logger.verbose(`Content length: ${content.length} characters`);
+            return { status: 'success', content };
+          } catch (navError) {
+            // If that fails, try with no waitUntil option
+            logger.warn(
+              `Failed with domcontentloaded strategy: ${errorToString(navError)}`,
+            );
+            logger.verbose(
+              `Retrying navigation to ${action.url} with no waitUntil option`,
+            );
+
+            try {
+              await page.goto(action.url);
+              await sleep(3000);
+              const content = await filterPageContent(page, pageFilter);
+              logger.verbose(`Content: ${content}`);
+              logger.verbose('Navigation completed with basic strategy');
+              return { status: 'success', content };
+            } catch (innerError) {
+              logger.error(
+                `Failed with basic navigation strategy: ${errorToString(innerError)}`,
+              );
+              throw innerError; // Re-throw to be caught by outer catch block
+            }
+          }
         }
 
         case 'click': {
@@ -102,7 +159,8 @@ export const browseMessageTool: Tool<Parameters, ReturnType> = {
             action.selectorType,
           );
           await page.click(clickSelector);
-          const content = await page.content();
+          await sleep(1000); // Wait for any content changes after click
+          const content = await filterPageContent(page, pageFilter);
           logger.verbose(
             `Click action completed on selector: ${clickSelector}`,
           );
@@ -136,8 +194,9 @@ export const browseMessageTool: Tool<Parameters, ReturnType> = {
         }
 
         case 'content': {
-          const content = await page.content();
+          const content = await filterPageContent(page, pageFilter);
           logger.verbose('Page content retrieved successfully');
+          logger.verbose(`Content length: ${content.length} characters`);
           return { status: 'success', content };
         }
 
@@ -164,9 +223,12 @@ export const browseMessageTool: Tool<Parameters, ReturnType> = {
     }
   },
 
-  logParameters: ({ action, description }, { logger }) => {
+  logParameters: (
+    { action, description },
+    { logger, pageFilter = 'simple' },
+  ) => {
     logger.info(
-      `Performing browser action: ${action.actionType}, ${description}`,
+      `Performing browser action: ${action.actionType} with ${pageFilter} processing, ${description}`,
     );
   },
 

packages/agent/src/tools/browser/browseStart.ts

@@ -5,12 +5,13 @@ import { zodToJsonSchema } from 'zod-to-json-schema';
 
 import { Tool } from '../../core/types.js';
 import { errorToString } from '../../utils/errorToString.js';
+import { sleep } from '../../utils/sleep.js';
 
+import { filterPageContent } from './filterPageContent.js';
 import { browserSessions } from './types.js';
 
 const parameterSchema = z.object({
   url: z.string().url().optional().describe('Initial URL to navigate to'),
-
   timeout: z
     .number()
     .optional()
@@ -40,23 +41,37 @@ export const browseStartTool: Tool<Parameters, ReturnType> = {
 
   execute: async (
     { url, timeout = 30000 },
-    { logger, headless = true },
+    { logger, headless, userSession, pageFilter },
   ): Promise<ReturnType> => {
     logger.verbose(`Starting browser session${url ? ` at ${url}` : ''}`);
+    logger.verbose(
+      `User session mode: ${userSession ? 'enabled' : 'disabled'}`,
+    );
+    logger.verbose(`Webpage processing mode: ${pageFilter}`);
 
     try {
       const instanceId = uuidv4();
 
       // Launch browser
-      const browser = await chromium.launch({
+      const launchOptions = {
         headless,
-      });
+      };
+
+      // Use system Chrome installation if userSession is true
+      if (userSession) {
+        logger.verbose('Using system Chrome installation');
+        // For Chrome, we use the channel option to specify Chrome
+        launchOptions['channel'] = 'chrome';
+      }
+
+      const browser = await chromium.launch(launchOptions);
 
       // Create new context with default settings
       const context = await browser.newContext({
         viewport: null,
         userAgent:
           'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+        serviceWorkers: 'block', // Block service workers which can cause continuous network activity
       });
 
       // Create new page
@@ -80,11 +95,42 @@ export const browseStartTool: Tool<Parameters, ReturnType> = {
       // Navigate to URL if provided
       let content = '';
       if (url) {
-        await page.goto(url, { waitUntil: 'networkidle' });
-        content = await page.content();
+        try {
+          // Try with 'domcontentloaded' first which is more reliable than 'networkidle'
+          logger.verbose(
+            `Navigating to ${url} with 'domcontentloaded' waitUntil`,
+          );
+          await page.goto(url, { waitUntil: 'domcontentloaded', timeout });
+          await sleep(3000);
+          content = await filterPageContent(page, pageFilter);
+          logger.verbose(`Content: ${content}`);
+          logger.verbose('Navigation completed with domcontentloaded strategy');
+        } catch (error) {
+          // If that fails, try with no waitUntil option at all (most basic)
+          logger.warn(
+            `Failed with domcontentloaded strategy: ${errorToString(error)}`,
+          );
+          logger.verbose(
+            `Retrying navigation to ${url} with no waitUntil option`,
+          );
+
+          try {
+            await page.goto(url, { timeout });
+            await sleep(3000);
+            content = await filterPageContent(page, pageFilter);
+            logger.verbose(`Content: ${content}`);
+            logger.verbose('Navigation completed with basic strategy');
+          } catch (innerError) {
+            logger.error(
+              `Failed with basic navigation strategy: ${errorToString(innerError)}`,
+            );
+            throw innerError; // Re-throw to be caught by outer catch block
+          }
+        }
       }
 
       logger.verbose('Browser session started successfully');
+      logger.verbose(`Content length: ${content.length} characters`);
 
       return {
         instanceId,
@@ -101,9 +147,9 @@ export const browseStartTool: Tool<Parameters, ReturnType> = {
     }
   },
 
-  logParameters: ({ url, description }, { logger }) => {
+  logParameters: ({ url, description }, { logger, pageFilter = 'simple' }) => {
     logger.info(
-      `Starting browser session${url ? ` at ${url}` : ''}, ${description}`,
+      `Starting browser session${url ? ` at ${url}` : ''} with ${pageFilter} processing, ${description}`,
     );
   },