chore: allow all/missing/none run agents modes (#38628)

Author: pavelfeldman

docs/src/test-api/class-fullconfig.md

@@ -106,9 +106,12 @@ Base directory for all relative paths used in the reporters.
 
 ## property: FullConfig.runAgents
 * since: v1.58
-- type: <boolean>
+- type: <['RunAgentsMode]<"all"|"missing"|"none">>
 
-Run agents to generate the code for [`method: Page.perform`] and similar.
+Whether to run LLM agent for [`method: Page.perform`]:
+* "all" disregards existing cache and performs all actions via LLM
+* "missing" only performs actions that don't have generated cache actions
+* "none" does not talk to LLM at all, relies on the cached actions (default)
 
 ## property: FullConfig.shard
 * since: v1.10

docs/src/test-api/class-testconfig.md

@@ -516,9 +516,12 @@ export default defineConfig({
 
 ## property: TestConfig.runAgents
 * since: v1.58
-- type: ?<boolean>
+- type: ?<['RunAgentsMode]<"all"|"missing"|"none">>
 
-Run agents to generate the code for [`method: Page.perform`] and similar.
+Whether to run LLM agent for [`method: Page.perform`]:
+* "all" disregards existing cache and performs all actions via LLM
+* "missing" only performs actions that don't have generated cache actions
+* "none" does not talk to LLM at all, relies on the cached actions (default)
 
 ## property: TestConfig.shard
 * since: v1.10

packages/playwright/src/common/config.ts

@@ -112,7 +112,7 @@ export class FullConfigInternal {
       quiet: takeFirst(configCLIOverrides.quiet, userConfig.quiet, false),
       reporter: takeFirst(configCLIOverrides.reporter, resolveReporters(userConfig.reporter, configDir), [[defaultReporter]]),
       reportSlowTests: takeFirst(userConfig.reportSlowTests, { max: 5, threshold: 300_000 /* 5 minutes */ }),
-      runAgents: takeFirst(configCLIOverrides.runAgents, userConfig.runAgents, false),
+      runAgents: takeFirst(configCLIOverrides.runAgents, userConfig.runAgents, 'none'),
       shard: takeFirst(configCLIOverrides.shard, userConfig.shard, null),
       tags: globalTags,
       updateSnapshots: takeFirst(configCLIOverrides.updateSnapshots, userConfig.updateSnapshots, 'missing'),

packages/playwright/src/common/ipc.ts

@@ -43,7 +43,7 @@ export type ConfigCLIOverrides = {
   ignoreSnapshots?: boolean;
   updateSnapshots?: 'all' | 'changed' | 'missing' | 'none';
   updateSourceMethod?: 'overwrite' | 'patch' | '3way';
-  runAgents?: boolean;
+  runAgents?: 'all' | 'missing' | 'none';
   workers?: number | string;
   projects?: { name: string, use?: any }[],
   use?: any;

packages/playwright/src/index.ts

@@ -716,9 +716,11 @@ class ArtifactsRecorder {
     this._agentCacheFile = this._testInfo._applyPathTemplate(cachePathTemplate, '', '.json');
     this._agentCacheOutFile = path.join(this._testInfo.artifactsDir(), 'agent-cache-' + createGuid() + '.json');
 
-    const cacheFile = this._testInfo.config.runAgents ? undefined : await this._testInfo._cloneStorage(this._agentCacheFile);
+    const cacheFile = this._testInfo.config.runAgents === 'all' ? undefined : await this._testInfo._cloneStorage(this._agentCacheFile);
     options.agent = {
       ...this._agent,
+      provider: this._testInfo.config.runAgents !== 'none' ? this._agent.provider : undefined,
+      model: this._testInfo.config.runAgents !== 'none' ? this._agent.model : undefined,
       cacheFile,
       cacheOutFile: this._agentCacheOutFile,
     };

packages/playwright/src/isomorphic/teleReceiver.ts

@@ -782,7 +782,7 @@ export const baseFullConfig: reporterTypes.FullConfig = {
   tags: [],
   updateSnapshots: 'missing',
   updateSourceMethod: 'patch',
-  runAgents: false,
+  runAgents: 'none',
   version: '',
   workers: 0,
   webServer: null,

packages/playwright/src/isomorphic/testServerInterface.ts

@@ -97,7 +97,7 @@ export interface TestServerInterface {
     workers?: number | string;
     updateSnapshots?: 'all' | 'changed' | 'missing' | 'none';
     updateSourceMethod?: 'overwrite' | 'patch' | '3way';
-    runAgents?: boolean;
+    runAgents?: 'all' | 'missing' | 'none';
     reporters?: string[],
     trace?: 'on' | 'off';
     video?: 'on' | 'off';

packages/playwright/src/program.ts

@@ -422,7 +422,7 @@ const testOptions: [string, { description: string, choices?: string[], preset?:
   ['--repeat-each <N>', { description: `Run each test N times (default: 1)` }],
   ['--reporter <reporter>', { description: `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${defaultReporter}")` }],
   ['--retries <retries>', { description: `Maximum retry count for flaky tests, zero for no retries (default: no retries)` }],
-  ['--run-agents', { description: `Run agents to generate the code for page.perform` }],
+  ['--run-agents <mode>', { description: `Run agents to generate the code for page.perform`, choices: ['missing', 'all', 'none'], preset: 'none' }],
   ['--shard <shard>', { description: `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"` }],
   ['--shard-weights <weights>', { description: `Weights for each shard, colon-separated, for example "2:1:1" for 3 shards where the first shard should be allocated half of the work` }],
   ['--test-list <file>', { description: `Path to a file containing a list of tests to run. See https://playwright.dev/docs/test-cli for more details.` }],

packages/playwright/src/runner/testRunner.ts

@@ -68,7 +68,7 @@ export type RunTestsParams = {
   workers?: number | string;
   updateSnapshots?: 'all' | 'changed' | 'missing' | 'none';
   updateSourceMethod?: 'overwrite' | 'patch' | '3way';
-  runAgents?: boolean;
+  runAgents?: 'all' | 'missing' | 'none';
   reporters?: string[],
   trace?: 'on' | 'off';
   video?: 'on' | 'off';

packages/playwright/types/test.d.ts

@@ -1612,10 +1612,13 @@ interface TestConfig<TestArgs = {}, WorkerArgs = {}> {
   retries?: number;
 
   /**
-   * Run agents to generate the code for
-   * [page.perform(task[, options])](https://playwright.dev/docs/api/class-page#page-perform) and similar.
+   * Whether to run LLM agent for
+   * [page.perform(task[, options])](https://playwright.dev/docs/api/class-page#page-perform):
+   * - "all" disregards existing cache and performs all actions via LLM
+   * - "missing" only performs actions that don't have generated cache actions
+   * - "none" does not talk to LLM at all, relies on the cached actions (default)
    */
-  runAgents?: boolean;
+  runAgents?: "all"|"missing"|"none";
 
   /**
    * Shard tests and execute only the selected shard. Specify in the one-based form like `{ total: 5, current: 2 }`.
@@ -2074,10 +2077,13 @@ export interface FullConfig<TestArgs = {}, WorkerArgs = {}> {
   rootDir: string;
 
   /**
-   * Run agents to generate the code for
-   * [page.perform(task[, options])](https://playwright.dev/docs/api/class-page#page-perform) and similar.
+   * Whether to run LLM agent for
+   * [page.perform(task[, options])](https://playwright.dev/docs/api/class-page#page-perform):
+   * - "all" disregards existing cache and performs all actions via LLM
+   * - "missing" only performs actions that don't have generated cache actions
+   * - "none" does not talk to LLM at all, relies on the cached actions (default)
    */
-  runAgents: boolean;
+  runAgents: "all"|"missing"|"none";
 
   /**
    * See [testConfig.shard](https://playwright.dev/docs/api/class-testconfig#test-config-shard).
@@ -6952,7 +6958,7 @@ export type Agent = {
   cachePathTemplate?: string;
   maxTurns?: number;
   maxTokens?: number;
-  runAgents?: boolean;
+  runAgents?: 'all' | 'missing' | 'none';
   secrets?: { [key: string]: string };
 };