feat(test-runner): add httpCache config option

docs/src/test-api/class-fullconfig.md

@@ -15,6 +15,14 @@ arguments — for example, args supplied after the `--` separator
 these; consumers are responsible for slicing and interpreting them with
 any argument-parsing library.
 
+## property: FullConfig.httpCache
+* since: v1.62
+- type: ?<[Object]|[Array]<[Object]>>
+  - `dir` <[string]>
+  - `match` ?<[string]|[RegExp]>
+
+See [`property: TestConfig.httpCache`].
+
 ## property: FullConfig.configFile
 * since: v1.20
 - type: ?<[string]>

docs/src/test-api/class-testconfig.md

@@ -68,6 +68,43 @@ The structure of the git commit metadata is subject to change.
 :::
 
 
+## property: TestConfig.httpCache
+* since: v1.62
+- type: ?<[Object]|[Array]<[Object]>>
+  - `dir` <[string]> Directory to store and replay the cache. Resolved relative to the configuration file.
+  - `match` ?<[string]|[RegExp]> Glob pattern or regular expression matched against the request URL to decide which requests are cached. When omitted, all requests are cached.
+
+Records responses to disk and replays them on subsequent runs. When set, Playwright
+starts a caching proxy for the run and routes all browser traffic through it: on the
+first run matching responses are recorded to `dir`, and on subsequent runs those requests
+are served from disk without hitting the network. All workers share the single proxy, so
+a resource fetched by one worker is a cache hit for the others.
+
+Pass an array of caches to route different requests to different directories. Each request
+is handled by the first cache whose `match` matches its URL, so list more specific caches
+first; requests that match no cache are passed through without caching.
+
+**Usage**
+
+```js title="playwright.config.ts"
+import { defineConfig } from '@playwright/test';
+
+export default defineConfig({
+  httpCache: { dir: './.network-cache', match: '**/assets/**' },
+});
+```
+
+```js title="playwright.config.ts"
+import { defineConfig } from '@playwright/test';
+
+export default defineConfig({
+  httpCache: [
+    { dir: './.cache/api', match: '**/api/**' },
+    { dir: './.cache/assets', match: '**/assets/**' },
+  ],
+});
+```
+
 ## property: TestConfig.expect
 * since: v1.10
 - type: ?<[Object]>

packages/playwright/src/common/config.ts

@@ -109,6 +109,7 @@ export class FullConfigInternal {
       version: packageJSON.version,
       workers: resolveWorkers(takeFirst((configCLIOverrides.debug || configCLIOverrides.pause) ? 1 : undefined, configCLIOverrides.workers, userConfig.workers, '50%')),
       webServer: null,
+      httpCache: resolveHttpCache(userConfig.httpCache, configDir),
     };
     for (const key in userConfig) {
       if (key.startsWith('@'))
@@ -218,6 +219,13 @@ function resolveReporters(reporters: Config['reporter'], rootDir: string): Repor
   });
 }
 
+function resolveHttpCache(httpCache: Config['httpCache'], configDir: string): FullConfig['httpCache'] {
+  if (!httpCache)
+    return undefined;
+  const resolve = <T extends { dir: string }>(entry: T): T => ({ ...entry, dir: path.resolve(configDir, entry.dir) });
+  return Array.isArray(httpCache) ? httpCache.map(resolve) : resolve(httpCache);
+}
+
 function resolveWorkers(workers: string | number): number {
   if (typeof workers === 'string') {
     if (workers.endsWith('%')) {

packages/playwright/src/index.ts

@@ -237,6 +237,8 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures, UtilityTestFixt
       options.headless = headless;
     if (channel !== undefined)
       options.channel = channel;
+    if (process.env.PLAYWRIGHT_TEST_CACHE_PROXY)
+      options.proxy = { server: process.env.PLAYWRIGHT_TEST_CACHE_PROXY, bypass: '<-loopback>' };
 
     playwright._defaultLaunchOptions = options;
     await use(options);
@@ -277,7 +279,7 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures, UtilityTestFixt
   geolocation: [({ contextOptions }, use) => use(contextOptions.geolocation), { option: true, box: true }],
   hasTouch: [({ contextOptions }, use) => use(contextOptions.hasTouch ?? false), { option: true, box: true }],
   httpCredentials: [({ contextOptions }, use) => use(contextOptions.httpCredentials), { option: true, box: true }],
-  ignoreHTTPSErrors: [({ contextOptions }, use) => use(contextOptions.ignoreHTTPSErrors ?? false), { option: true, box: true }],
+  ignoreHTTPSErrors: [({ contextOptions }, use) => use(process.env.PLAYWRIGHT_TEST_CACHE_PROXY ? true : (contextOptions.ignoreHTTPSErrors ?? false)), { option: true, box: true }],
   isMobile: [({ contextOptions }, use) => use(contextOptions.isMobile ?? false), { option: true, box: true }],
   javaScriptEnabled: [({ contextOptions }, use) => use(contextOptions.javaScriptEnabled ?? true), { option: true, box: true }],
   locale: [({ contextOptions }, use) => use(contextOptions.locale ?? 'en-US'), { option: true, box: true }],

packages/playwright/src/plugins/DEPS.list

@@ -1,5 +1,6 @@
 [*]
 @isomorphic/**
 @utils/**
+cacheProxy/**
 node_modules/colors/safe
 node_modules/debug

packages/playwright/src/plugins/cacheProxy/cache.ts

@@ -0,0 +1,179 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import crypto from 'crypto';
+import fs from 'fs';
+import path from 'path';
+
+const CACHE_VERSION = 1;
+const INLINE_THRESHOLD = 8 * 1024;
+
+export type CachedResponse = {
+  status: number;
+  statusText: string;
+  headers: [string, string][];
+  body: Buffer;
+};
+
+type CacheRecord = {
+  k: string;                       // key: sha1(method + '\n' + url)
+  u: string;                       // url, verbatim so the index stays greppable
+  s: number;                       // status
+  st: string;                      // statusText
+  hh: [string, string][];          // headers, verbatim (incl. content-encoding)
+  ts: number;                      // record time, epoch seconds
+  c?: string;                      // inline body (base64), or...
+  f?: string;                      // ...content-addressed blob (sha1)
+};
+
+export class ResponseCache {
+  private _dir: string;
+  private _blobsDir: string;
+  private _indexFile: string;
+  private _metaFile: string;
+  private _index = new Map<string, CacheRecord>();
+  private _writeChain: Promise<void> = Promise.resolve();
+  private _initialized = false;
+
+  constructor(dir: string) {
+    this._dir = path.resolve(dir);
+    this._blobsDir = path.join(this._dir, 'blobs');
+    this._indexFile = path.join(this._dir, 'index.jsonl');
+    this._metaFile = path.join(this._dir, 'meta.json');
+  }
+
+  async load() {
+    let content: string;
+    try {
+      content = await fs.promises.readFile(this._indexFile, 'utf8');
+    } catch {
+      return; // No cache yet.
+    }
+    // Discard an incompatible cache instead of mis-parsing it.
+    if (await this._onDiskVersion() !== CACHE_VERSION) {
+      await fs.promises.rm(this._indexFile, { force: true }).catch(() => {});
+      await fs.promises.rm(this._blobsDir, { recursive: true, force: true }).catch(() => {});
+      return;
+    }
+    for (const line of content.split('\n')) {
+      if (!line.trim())
+        continue;
+      try {
+        const record = JSON.parse(line) as CacheRecord;
+        this._index.set(record.k, record);
+      } catch {
+        // Ignore a malformed line rather than failing the whole cache.
+      }
+    }
+  }
+
+  static key(method: string, url: string): string {
+    return crypto.createHash('sha1').update(`${method}\n${url}`).digest('hex');
+  }
+
+  async get(key: string): Promise<CachedResponse | undefined> {
+    const record = this._index.get(key);
+    if (!record)
+      return undefined;
+    try {
+      const body = record.c !== undefined ? Buffer.from(record.c, 'base64') : await fs.promises.readFile(this._blobPath(record.f!));
+      return { status: record.s, statusText: record.st, headers: record.hh, body };
+    } catch {
+      return undefined; // Corrupt/missing blob - treat as a miss.
+    }
+  }
+
+  async set(key: string, url: string, response: CachedResponse) {
+    if (this._index.has(key))
+      return;
+    const record: CacheRecord = {
+      k: key,
+      u: url,
+      s: response.status,
+      st: response.statusText,
+      hh: response.headers,
+      ts: Math.floor(Date.now() / 1000),
+    };
+    if (response.body.length >= INLINE_THRESHOLD) {
+      const hash = contentHash(response.body);
+      await this._writeBlob(hash, response.body);
+      record.f = hash;
+    } else {
+      record.c = response.body.toString('base64');
+    }
+    this._index.set(key, record);
+    await this._append(record);
+  }
+
+  async flush() {
+    await this._writeChain;
+  }
+
+  private _blobPath(hash: string): string {
+    return path.join(this._blobsDir, hash.slice(0, 2), hash);
+  }
+
+  private async _writeBlob(hash: string, body: Buffer) {
+    const dest = this._blobPath(hash);
+    if (await exists(dest))
+      return;
+    await fs.promises.mkdir(path.dirname(dest), { recursive: true });
+    const tmp = `${dest}.tmp-${crypto.randomBytes(8).toString('hex')}`;
+    await fs.promises.writeFile(tmp, body);
+    try {
+      await fs.promises.rename(tmp, dest);
+    } catch {
+      await fs.promises.rm(tmp, { force: true }).catch(() => {});
+    }
+  }
+
+  private _append(record: CacheRecord): Promise<void> {
+    const line = JSON.stringify(record) + '\n';
+    const run = async () => {
+      if (!this._initialized) {
+        await fs.promises.mkdir(this._dir, { recursive: true });
+        await fs.promises.writeFile(this._metaFile, JSON.stringify({ version: CACHE_VERSION }, null, 2), { flag: 'w' });
+        this._initialized = true;
+      }
+      await fs.promises.appendFile(this._indexFile, line);
+    };
+    const result = this._writeChain.then(run, run);
+    this._writeChain = result.catch(() => {});
+    return result;
+  }
+
+  private async _onDiskVersion(): Promise<number | undefined> {
+    try {
+      const meta = JSON.parse(await fs.promises.readFile(this._metaFile, 'utf8'));
+      return typeof meta.version === 'number' ? meta.version : undefined;
+    } catch {
+      return undefined;
+    }
+  }
+}
+
+function contentHash(body: Buffer): string {
+  return crypto.createHash('sha1').update(body).digest('hex');
+}
+
+async function exists(file: string): Promise<boolean> {
+  try {
+    await fs.promises.access(file);
+    return true;
+  } catch {
+    return false;
+  }
+}