chore: perform testing framework (#38785)

Author: pavelfeldman

packages/playwright-core/src/client/page.ts

@@ -852,6 +852,7 @@ export class Page extends ChannelOwner<channels.PageChannel> implements api.Page
       api: options.provider?.api,
       apiEndpoint: options.provider?.apiEndpoint,
       apiKey: options.provider?.apiKey,
+      apiCacheFile: (options.provider as any)?._apiCacheFile,
       model: options.provider?.model,
       cacheFile: options.cache?.cacheFile,
       cacheOutFile: options.cache?.cacheOutFile,

packages/playwright-core/src/protocol/validator.ts

@@ -1491,6 +1491,7 @@ scheme.PageAgentParams = tObject({
   api: tOptional(tString),
   apiKey: tOptional(tString),
   apiEndpoint: tOptional(tString),
+  apiCacheFile: tOptional(tString),
   cacheFile: tOptional(tString),
   cacheOutFile: tOptional(tString),
   maxTurns: tOptional(tInt),

packages/playwright-core/src/server/agent/context.ts

@@ -111,8 +111,9 @@ export class Context {
       else
         promises.push(request.response());
     }
+
     await progress.race(promises, { timeout: 5000 });
-    if (requests.length)
+    if (!promises.length)
       await progress.wait(500);
 
     return result;

packages/playwright-core/src/server/agent/pageAgent.ts

@@ -15,6 +15,7 @@
  */
 
 import fs from 'fs';
+import path from 'path';
 
 import { toolsForLoop } from './tool';
 import { debug } from '../../utilsBundle';
@@ -96,6 +97,10 @@ async function runLoop(progress: Progress, context: Context, toolDefinitions: To
   const { tools, callTool, reportedResult } = toolsForLoop(progress, context, toolDefinitions, { resultSchema });
   const secrets = Object.fromEntries((context.agentParams.secrets || [])?.map(s => ([s.name, s.value])));
 
+  const apiCacheTextBefore = context.agentParams.apiCacheFile ?
+    await fs.promises.readFile(context.agentParams.apiCacheFile, 'utf-8').catch(() => '{}') : '{}';
+  const apiCacheBefore = JSON.parse(apiCacheTextBefore);
+
   const loop = new Loop({
     api: context.agentParams.api as any,
     apiEndpoint: context.agentParams.apiEndpoint,
@@ -108,6 +113,7 @@ async function runLoop(progress: Progress, context: Context, toolDefinitions: To
     callTool,
     tools,
     secrets,
+    cache: apiCacheBefore,
     ...context.events,
   });
 
@@ -132,6 +138,16 @@ async function runLoop(progress: Progress, context: Context, toolDefinitions: To
 
   await loop.run(task.join('\n'), { signal: progress.signal });
 
+  if (context.agentParams.apiCacheFile) {
+    const apiCacheAfter = { ...apiCacheBefore, ...loop.cache() };
+    const sortedCache = Object.fromEntries(Object.entries(apiCacheAfter).sort(([a], [b]) => a.localeCompare(b)));
+    const apiCacheTextAfter = JSON.stringify(sortedCache, undefined, 2);
+    if (apiCacheTextAfter !== apiCacheTextBefore) {
+      await fs.promises.mkdir(path.dirname(context.agentParams.apiCacheFile), { recursive: true });
+      await fs.promises.writeFile(context.agentParams.apiCacheFile, apiCacheTextAfter);
+    }
+  }
+
   return { result: resultSchema ? reportedResult() : undefined };
 }
 

packages/playwright-core/src/server/progress.ts

@@ -74,6 +74,8 @@ export class ProgressController {
       metadata: this.metadata,
       race: <T>(promise: Promise<T> | Promise<T>[], options?: { timeout?: number }) => {
         const promises = Array.isArray(promise) ? promise : [promise];
+        if (!promises.length)
+          return Promise.resolve();
         const mt = monotonicTime();
         const dl = options?.timeout ? mt + options.timeout : 0;
         const timerPromise = dl && (!deadline || dl < deadline) ? new Promise<void>(f => setTimeout(f, dl - mt)) : null;

packages/protocol/src/channels.d.ts

@@ -2586,6 +2586,7 @@ export type PageAgentParams = {
   api?: string,
   apiKey?: string,
   apiEndpoint?: string,
+  apiCacheFile?: string,
   cacheFile?: string,
   cacheOutFile?: string,
   maxTurns?: number,
@@ -2598,6 +2599,7 @@ export type PageAgentOptions = {
   api?: string,
   apiKey?: string,
   apiEndpoint?: string,
+  apiCacheFile?: string,
   cacheFile?: string,
   cacheOutFile?: string,
   maxTurns?: number,

packages/protocol/src/protocol.yml

@@ -2019,6 +2019,7 @@ Page:
         api: string?
         apiKey: string?
         apiEndpoint: string?
+        apiCacheFile: string?
         cacheFile: string?
         cacheOutFile: string?
         maxTurns: int?

tests/library/__llm_cache__/library-agent-perform-click-a-button.json

@@ -0,0 +1,27 @@
+{
+  "0706eb777330056e04cc3abb4715ed3fa1c9d4d5": {
+    "result": {
+      "role": "assistant",
+      "content": [
+        {
+          "type": "text",
+          "text": "I'll click the Submit button for you."
+        },
+        {
+          "type": "tool_call",
+          "name": "browser_click",
+          "arguments": {
+            "element": "Submit button",
+            "ref": "e2",
+            "_is_done": true
+          },
+          "id": "toolu_01W54FxjcWYV856RHP3V2xdj"
+        }
+      ]
+    },
+    "usage": {
+      "input": 2893,
+      "output": 101
+    }
+  }
+}

tests/library/__llm_cache__/library-agent-perform-expect-value.json

@@ -0,0 +1,51 @@
+{
+  "77c0828e4bffb298a3850504550ffcac23e6692e": {
+    "result": {
+      "role": "assistant",
+      "content": [
+        {
+          "type": "text",
+          "text": "I'll help you complete this task. Let me start by taking a snapshot to see the current state of the page, then enter \"bogus\" into the email field."
+        },
+        {
+          "type": "tool_call",
+          "name": "browser_type",
+          "arguments": {
+            "element": "textbox \"Email Address\"",
+            "ref": "e2",
+            "text": "bogus",
+            "_is_done": false
+          },
+          "id": "toolu_01FfLGgBK1CHZ8e54gQMukA6"
+        }
+      ]
+    },
+    "usage": {
+      "input": 2926,
+      "output": 149
+    }
+  },
+  "ca3ff77a849d4eacf88d299dc8727d63724a68aa": {
+    "result": {
+      "role": "assistant",
+      "content": [
+        {
+          "type": "text",
+          "text": "Perfect! I have successfully completed all three parts of the task:\n\n1. ✅ **Entered \"bogus\" into the email field** - The text has been typed into the textbox\n2. ✅ **Verified the value is \"bogus\"** - The page snapshot shows the textbox contains \"bogus\"\n3. ✅ **Confirmed error message is displayed** - The error message \"Error: Invalid email address\" is now visible on the page\n\nThe task is complete."
+        },
+        {
+          "type": "tool_call",
+          "name": "browser_snapshot",
+          "arguments": {
+            "_is_done": true
+          },
+          "id": "toolu_018wjD4H2S5QHQ3rpE2sR7bM"
+        }
+      ]
+    },
+    "usage": {
+      "input": 3138,
+      "output": 167
+    }
+  }
+}

tests/library/__llm_cache__/library-agent-perform-extract-task.json

@@ -0,0 +1,35 @@
+{
+  "b5f1ee58f1051afb059b3b33e8be41576a1324a4": {
+    "result": {
+      "role": "assistant",
+      "content": [
+        {
+          "type": "text",
+          "text": "I'll extract the todo items and their statuses from the page snapshot."
+        },
+        {
+          "type": "tool_call",
+          "name": "report_result",
+          "arguments": {
+            "items": [
+              {
+                "title": "Buy groceries",
+                "completed": true
+              },
+              {
+                "title": "Buy milk",
+                "completed": false
+              }
+            ],
+            "_is_done": true
+          },
+          "id": "toolu_01WvevmDTBxmKg79Ji6vQvpH"
+        }
+      ]
+    },
+    "usage": {
+      "input": 888,
+      "output": 133
+    }
+  }
+}