[RPP] Compute estimated savings for Duplicated JavaScript insight

- Ported Lighthouse's estimateCompressedContentSize
- Ported Lightouse's isRequestCompressed - this was inlined within the
  DocumentLatency insight, except it did not handle Lightrider/PSI. I
  changed that insight to call the new helper function.
- Moved the part of Trace Processor that deferred resolving cached maps
  to the provided `resolveSourceMap` option, to instead be handled
  internally within ScriptsHandler. In these cases, resolveSourceMap is
  still called w/ the cached raw source map, but all it does is create
  an SDK.SourceMap.
- The testing/TraceLoader was not resolving cached source maps stored
  in the metadata. Now it is.
- Fixed a crash when loading an enhanced trace containing a source map
  that cannot be encoded via btoa. For now, just ignore the error.

Bug: 394373632
Change-Id: I49104aee7fbe5f4b9dfc5061973b7e941f3a2911
Reviewed-on: https://chromium-review.googlesource.com/c/devtools/devtools-frontend/+/6404895
Auto-Submit: Connor Clark <[email protected]>
Reviewed-by: Paul Irish <[email protected]>
Commit-Queue: Connor Clark <[email protected]>
Commit-Queue: Paul Irish <[email protected]>

Author: Connor Clark

front_end/core/sdk/EnhancedTracesParser.ts

@@ -234,7 +234,12 @@ export class EnhancedTracesParser {
       return;
     }
 
-    return `data:text/plain;base64,${btoa(JSON.stringify(sourceMap))}`;
+    try {
+      return `data:text/plain;base64,${btoa(JSON.stringify(sourceMap))}`;
+    } catch {
+      // TODO(cjamcl): getting InvalidCharacterError (try loading dupe-js.json.gz).
+      return;
+    }
   }
 
   private getSourceMapFromMetadata(script: RehydratingScript): SourceMapV3|undefined {

front_end/models/trace/handlers/ScriptsHandler.ts

@@ -200,6 +200,25 @@ export function getScriptGeneratedSizes(script: Script): GeneratedFileSizes|null
   return script.sizes ?? null;
 }
 
+function findCachedRawSourceMap(
+    sourceMapUrl: string, options: Types.Configuration.ParseOptions): SDK.SourceMap.SourceMapV3|undefined {
+  if (!sourceMapUrl) {
+    return;
+  }
+
+  // If loading from disk, check the metadata for source maps.
+  // The metadata doesn't store data url source maps.
+  const isDataUrl = sourceMapUrl.startsWith('data:');
+  if (!options.isFreshRecording && options.metadata?.sourceMaps && !isDataUrl) {
+    const cachedSourceMap = options.metadata.sourceMaps.find(m => m.sourceMapUrl === sourceMapUrl);
+    if (cachedSourceMap) {
+      return cachedSourceMap.sourceMap;
+    }
+  }
+
+  return;
+}
+
 export async function finalize(options: Types.Configuration.ParseOptions): Promise<void> {
   const networkRequests = [...networkRequestsHandlerData().byId.values()];
   for (const script of scriptById.values()) {
@@ -248,6 +267,7 @@ export async function finalize(options: Types.Configuration.ParseOptions): Promi
       scriptUrl: sourceUrl as Platform.DevToolsPath.UrlString,
       sourceMapUrl: sourceMapUrl as Platform.DevToolsPath.UrlString,
       frame: script.frame as Protocol.Page.FrameId,
+      cachedRawSourceMap: findCachedRawSourceMap(sourceMapUrl, options),
     };
     const promise = options.resolveSourceMap(params).then(sourceMap => {
       if (sourceMap) {

front_end/models/trace/insights/BUILD.gn

@@ -59,6 +59,7 @@ ts_library("unittests") {
     "Common.test.ts",
     "DOMSize.test.ts",
     "DocumentLatency.test.ts",
+    "DuplicatedJavaScript.test.ts",
     "FontDisplay.test.ts",
     "ForcedReflow.test.ts",
     "ImageDelivery.test.ts",

front_end/models/trace/insights/Common.test.ts

@@ -2,12 +2,15 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+import type {RecursivePartial} from '../../../core/platform/TypescriptUtilities.js';
+import * as Protocol from '../../../generated/protocol.js';
 import {describeWithEnvironment} from '../../../testing/EnvironmentHelpers.js';
 import {getFirstOrError, processTrace} from '../../../testing/InsightHelpers.js';
+import type * as Types from '../types/types.js';
 
 import * as Insights from './insights.js';
 
-const {calculateMetricWeightsForSorting} = Insights.Common;
+const {calculateMetricWeightsForSorting, estimateCompressedContentSize} = Insights.Common;
 
 describeWithEnvironment('Common', function() {
   describe('calculateMetricWeightsForSorting', () => {
@@ -53,4 +56,69 @@ describeWithEnvironment('Common', function() {
       assert.deepEqual(weights, {lcp: 0.48649783990559314, inp: 0.48649783990559314, cls: 0.027004320188813675});
     });
   });
+
+  describe('#estimateCompressedContentSize', () => {
+    const estimate = estimateCompressedContentSize;
+    const encoding = [{name: 'Content-Encoding', value: 'gzip'}];
+    const makeRequest = (partial: {
+                          transferSize?: number,
+                          resourceSize?: number, resourceType: Protocol.Network.ResourceType,
+                          responseHeaders?: Array<{name: string, value: string}>,
+                        }): Types.Events.SyntheticNetworkRequest => {
+      const request: RecursivePartial<Types.Events.SyntheticNetworkRequest> = {
+        args: {
+          data: {
+            encodedDataLength: partial.transferSize ?? 0,
+            decodedBodyLength: partial.resourceSize ?? 0,
+            resourceType: partial.resourceType,
+            responseHeaders: partial.responseHeaders ?? [],
+          }
+        },
+      };
+      return request as Types.Events.SyntheticNetworkRequest;
+    };
+
+    it('should estimate by resource type compression ratio when no network info available', () => {
+      assert.strictEqual(estimate(undefined, 1000, Protocol.Network.ResourceType.Stylesheet), 200);
+      assert.strictEqual(estimate(undefined, 1000, Protocol.Network.ResourceType.Script), 330);
+      assert.strictEqual(estimate(undefined, 1000, Protocol.Network.ResourceType.Document), 330);
+      assert.strictEqual(estimate(undefined, 1000, Protocol.Network.ResourceType.Other), 500);
+    });
+
+    it('should return transferSize when asset matches and is encoded', () => {
+      const resourceType = Protocol.Network.ResourceType.Stylesheet;
+      const request = makeRequest({transferSize: 1234, resourceType, responseHeaders: encoding});
+      const result = estimate(request, 10000, resourceType);
+      assert.strictEqual(result, 1234);
+    });
+
+    it('should return resourceSize when asset matches and is not encoded', () => {
+      const resourceType = Protocol.Network.ResourceType.Stylesheet;
+      const request = makeRequest({transferSize: 1235, resourceSize: 1234, resourceType, responseHeaders: []});
+      const result = estimate(request, 10000, resourceType);
+      assert.strictEqual(result, 1234);
+    });
+
+    // Ex: JS script embedded in HTML response.
+    it('should estimate by network compression ratio when asset does not match', () => {
+      const resourceType = Protocol.Network.ResourceType.Other;
+      const request = makeRequest({resourceSize: 2000, transferSize: 1000, resourceType, responseHeaders: encoding});
+      const result = estimate(request, 100, Protocol.Network.ResourceType.Script);
+      assert.strictEqual(result, 50);
+    });
+
+    it('should not error when missing resource size', () => {
+      const resourceType = Protocol.Network.ResourceType.Other;
+      const request = makeRequest({transferSize: 1000, resourceType, responseHeaders: []});
+      const result = estimate(request, 100, Protocol.Network.ResourceType.Script);
+      assert.strictEqual(result, 100);
+    });
+
+    it('should not error when resource size is 0', () => {
+      const resourceType = Protocol.Network.ResourceType.Other;
+      const request = makeRequest({transferSize: 1000, resourceSize: 0, resourceType, responseHeaders: []});
+      const result = estimate(request, 100, Protocol.Network.ResourceType.Script);
+      assert.strictEqual(result, 100);
+    });
+  });
 });

front_end/models/trace/insights/Common.ts

@@ -2,7 +2,9 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+import * as Protocol from '../../../generated/protocol.js';
 import type * as CrUXManager from '../../crux-manager/crux-manager.js';
+import type * as Handlers from '../handlers/handlers.js';
 import * as Helpers from '../helpers/helpers.js';
 import type * as Lantern from '../lantern/lantern.js';
 import * as Types from '../types/types.js';
@@ -293,3 +295,98 @@ export function metricSavingsForWastedBytes(
     LCP: estimateSavingsWithGraphs(wastedBytesByRequestId, simulator, lcpGraph),
   };
 }
+
+/**
+ * Returns whether the network request was sent encoded.
+ */
+export function isRequestCompressed(request: Types.Events.SyntheticNetworkRequest): boolean {
+  // FYI: In Lighthouse, older devtools logs (like our test fixtures) seems to be
+  // lower case, while modern logs are Cased-Like-This.
+  const patterns = [
+    /^content-encoding$/i, /^x-content-encoding-over-network$/i,
+    /^x-original-content-encoding$/i,  // Lightrider.
+  ];
+  const compressionTypes = ['gzip', 'br', 'deflate', 'zstd'];
+  return request.args.data.responseHeaders.some(
+      header => patterns.some(p => header.name.match(p)) && compressionTypes.includes(header.value));
+}
+
+function getRequestSizes(request: Types.Events.SyntheticNetworkRequest): {resourceSize: number, transferSize: number} {
+  const resourceSize = request.args.data.decodedBodyLength;
+  const transferSize = request.args.data.encodedDataLength;
+  return {resourceSize, transferSize};
+}
+
+/**
+ * Estimates the number of bytes the content of this network record would have consumed on the network based on the
+ * uncompressed size (totalBytes). Uses the actual transfer size from the network record if applicable,
+ * minus the size of the response headers.
+ *
+ * @param totalBytes Uncompressed size of the resource
+ */
+export function estimateCompressedContentSize(
+    request: Types.Events.SyntheticNetworkRequest|undefined, totalBytes: number,
+    resourceType: Protocol.Network.ResourceType): number {
+  if (!request) {
+    // We don't know how many bytes this asset used on the network, but we can guess it was
+    // roughly the size of the content gzipped.
+    // See https://developers.google.com/web/fundamentals/performance/optimizing-content-efficiency/optimize-encoding-and-transfer for specific CSS/Script examples
+    // See https://discuss.httparchive.org/t/file-size-and-compression-savings/145 for fallback multipliers
+    switch (resourceType) {
+      case 'Stylesheet':
+        // Stylesheets tend to compress extremely well.
+        return Math.round(totalBytes * 0.2);
+      case 'Script':
+      case 'Document':
+        // Scripts and HTML compress fairly well too.
+        return Math.round(totalBytes * 0.33);
+      default:
+        // Otherwise we'll just fallback to the average savings in HTTPArchive
+        return Math.round(totalBytes * 0.5);
+    }
+  }
+
+  // Get the size of the response body on the network.
+  const {transferSize, resourceSize} = getRequestSizes(request);
+  let contentTransferSize = transferSize;
+  if (!isRequestCompressed(request)) {
+    // This is not compressed, so we can use resourceSize directly.
+    // This would be equivalent to transfer size minus headers transfer size, but transfer size
+    // may also include bytes for SSL connection etc.
+    contentTransferSize = resourceSize;
+  }
+  // TODO(cjamcl): Get "responseHeadersTransferSize" in Network handler.
+  // else if (request.responseHeadersTransferSize) {
+  //   // Subtract the size of the encoded headers.
+  //   contentTransferSize =
+  //     Math.max(0, contentTransferSize - request.responseHeadersTransferSize);
+  // }
+
+  if (request.args.data.resourceType === resourceType) {
+    // This was a regular standalone asset, just use the transfer size.
+    return contentTransferSize;
+  }
+
+  // This was an asset that was inlined in a different resource type (e.g. HTML document).
+  // Use the compression ratio of the resource to estimate the total transferred bytes.
+  // Get the compression ratio, if it's an invalid number, assume no compression.
+  const compressionRatio = Number.isFinite(resourceSize) && resourceSize > 0 ? (contentTransferSize / resourceSize) : 1;
+  return Math.round(totalBytes * compressionRatio);
+}
+
+/**
+ * Utility function to estimate the ratio of the compression of a script.
+ * This excludes the size of the response headers.
+ */
+export function estimateCompressionRatioForScript(script: Handlers.ModelHandlers.Scripts.Script): number {
+  if (!script.request) {
+    // Can't find request, so just use 1.
+    return 1;
+  }
+
+  const request = script.request;
+  const contentLength = request.args.data.decodedBodyLength ?? script.content?.length ?? 0;
+  const compressedSize = estimateCompressedContentSize(request, contentLength, Protocol.Network.ResourceType.Script);
+  const compressionRatio = compressedSize / contentLength;
+  return compressionRatio;
+}

front_end/models/trace/insights/DocumentLatency.ts

@@ -7,6 +7,7 @@ import type * as Handlers from '../handlers/handlers.js';
 import * as Helpers from '../helpers/helpers.js';
 import * as Types from '../types/types.js';
 
+import {isRequestCompressed} from './Common.js';
 import {
   type Checklist,
   InsightCategory,
@@ -101,15 +102,7 @@ function getServerResponseTime(request: Types.Events.SyntheticNetworkRequest): T
 }
 
 function getCompressionSavings(request: Types.Events.SyntheticNetworkRequest): number {
-  // Check from headers if compression was already applied.
-  // Older devtools logs are lower case, while modern logs are Cased-Like-This.
-  const patterns = [
-    /^content-encoding$/i,
-    /^x-content-encoding-over-network$/i,
-  ];
-  const compressionTypes = ['gzip', 'br', 'deflate', 'zstd'];
-  const isCompressed = request.args.data.responseHeaders.some(
-      header => patterns.some(p => header.name.match(p)) && compressionTypes.includes(header.value));
+  const isCompressed = isRequestCompressed(request);
   if (isCompressed) {
     return 0;
   }

front_end/models/trace/insights/DuplicatedJavaScript.test.ts

@@ -0,0 +1,49 @@
+// Copyright 2025 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import {describeWithEnvironment} from '../../../testing/EnvironmentHelpers.js';
+import {getFirstOrError, getInsightOrError, processTrace} from '../../../testing/InsightHelpers.js';
+import type * as Trace from '../trace.js';
+
+describeWithEnvironment('DuplicatedJavaScript', function() {
+  it('works', async () => {
+    const {data, insights} = await processTrace(this, 'dupe-js.json.gz');
+    assert.strictEqual(insights.size, 1);
+    const insight = getInsightOrError(
+        'DuplicatedJavaScript', insights, getFirstOrError(data.Meta.navigationsByNavigationId.values()));
+
+    const duplication = insight.duplicationGroupedByNodeModules;
+    const results = Object.fromEntries(
+        [...duplication.entries()].filter(v => v[1].estimatedDuplicateBytes > 1000 * 100).map(([key, data]) => {
+          return [key, data.duplicates.map(v => ({url: v.script.url, resourceSize: v.attributedSize}))];
+        }));
+    const url1 = 'https://dupe-modules-lh-2.surge.sh/bundle.js?v1';
+    const url2 = 'https://dupe-modules-lh-2.surge.sh/bundle.js?v2';
+    const url3 = 'https://dupe-modules-lh-2.surge.sh/bundle.js?v3';
+    const url4 = 'https://dupe-modules-lh-2.surge.sh/bundle.js?v4';
+
+    assert.deepEqual(results, {
+      'node_modules/@headlessui/react': [
+        {url: url1, resourceSize: 56331},
+        {url: url2, resourceSize: 56331},
+        {url: url3, resourceSize: 56331},
+        {url: url4, resourceSize: 56331},
+      ],
+      'node_modules/filestack-js': [
+        {url: url1, resourceSize: 423206},
+        {url: url2, resourceSize: 423206},
+        {url: url3, resourceSize: 423206},
+        {url: url4, resourceSize: 423206},
+      ],
+      'node_modules/react-query': [
+        {url: url1, resourceSize: 40357},
+        {url: url2, resourceSize: 40357},
+        {url: url3, resourceSize: 40357},
+        {url: url4, resourceSize: 40357},
+      ],
+    });
+
+    assert.deepEqual(insight.metricSavings, {FCP: 100, LCP: 100} as Trace.Insights.Types.MetricSavings);
+  });
+});

front_end/models/trace/insights/DuplicatedJavaScript.ts

@@ -7,6 +7,7 @@ import * as Extras from '../extras/extras.js';
 import type * as Handlers from '../handlers/handlers.js';
 import * as Helpers from '../helpers/helpers.js';
 
+import {estimateCompressionRatioForScript, metricSavingsForWastedBytes} from './Common.js';
 import {
   InsightCategory,
   InsightKeys,
@@ -77,11 +78,28 @@ export function generateInsight(
 
   const {duplication, duplicationGroupedByNodeModules} = Extras.ScriptDuplication.computeScriptDuplication({scripts});
   const scriptsWithDuplication = [...duplication.values().flatMap(data => data.duplicates.map(d => d.script))];
+
+  const wastedBytesByRequestId = new Map<string, number>();
+  for (const {duplicates} of duplication.values()) {
+    for (let i = 1; i < duplicates.length; i++) {
+      const sourceData = duplicates[i];
+      if (!sourceData.script.request) {
+        continue;
+      }
+
+      const compressionRatio = estimateCompressionRatioForScript(sourceData.script);
+      const transferSize = Math.round(sourceData.attributedSize * compressionRatio);
+      const requestId = sourceData.script.request.args.data.requestId;
+      wastedBytesByRequestId.set(requestId, (wastedBytesByRequestId.get(requestId) || 0) + transferSize);
+    }
+  }
+
   return finalize({
     duplication,
     duplicationGroupedByNodeModules,
     scriptsWithDuplication: [...new Set(scriptsWithDuplication)],
     scripts,
     mainDocumentUrl: context.navigation?.args.data?.url ?? parsedTrace.Meta.mainFrameURL,
+    metricSavings: metricSavingsForWastedBytes(wastedBytesByRequestId, context),
   });
 }

front_end/models/trace/types/Configuration.ts

@@ -82,4 +82,6 @@ export interface ResolveSourceMapParams {
   scriptUrl: Platform.DevToolsPath.UrlString;
   sourceMapUrl: Platform.DevToolsPath.UrlString;
   frame: Protocol.Page.FrameId;
+  /** Set only if the raw source map was found on the provided metadata. Never set for source maps from data urls. */
+  cachedRawSourceMap?: SDK.SourceMap.SourceMapV3;
 }

front_end/panels/timeline/TimelinePanel.ts

@@ -2486,7 +2486,11 @@ export class TimelinePanel extends UI.Panel.Panel implements Client, TimelineMod
     }
 
     return async function resolveSourceMap(params: Trace.Types.Configuration.ResolveSourceMapParams) {
-      const {scriptId, scriptUrl, sourceMapUrl, frame} = params;
+      const {scriptId, scriptUrl, sourceMapUrl, frame, cachedRawSourceMap} = params;
+
+      if (cachedRawSourceMap) {
+        return new SDK.SourceMap.SourceMap(scriptUrl, sourceMapUrl, cachedRawSourceMap);
+      }
 
       // For still-active frames, the source map is likely already fetched or at least in-flight.
       if (isFreshRecording) {