[RPP] Add LH Treemap button to Duplicated JavaScript insight

This lifts a bunch of code from Lighthouse[1] needed for opening
the Lighthouse Treemap webapp[2], a treemap visualization tool
for JS bundles. It shows all the scripts found on the page,
augmented with source map and any other data we send it (like
what large modules are duplicated on the page).

A couple refactors were necessary:

- The DuplicatedJavaScript model has a few new properties
  needed to assemble the data for the treemap
- computeGeneratedFileSizes moved from ScriptDuplication to
  ScriptsHandler, so it can be cached
- Add `inline` property to Script trace model

[1] https://github.com/GoogleChrome/lighthouse/blob/04f43865487b77384d5bfb02f774397ead5e3db2/core/audits/script-treemap-data.js
[1] https://github.com/GoogleChrome/lighthouse/blob/04f43865487b77384d5bfb02f774397ead5e3db2/report/renderer/open-tab.js#L102C10-L102C18
[2] https://googlechrome.github.io/lighthouse/treemap/

Bug: 394373632
Change-Id: I1c873e298d8ff0611f325a1752110d7682182f5e
Reviewed-on: https://chromium-review.googlesource.com/c/devtools/devtools-frontend/+/6398204
Commit-Queue: Connor Clark <[email protected]>
Reviewed-by: Paul Irish <[email protected]>

Author: Connor Clark

config/gni/devtools_grd_files.gni

@@ -1983,6 +1983,7 @@ grd_files_debug_sources = [
   "front_end/panels/timeline/utils/ImageCache.js",
   "front_end/panels/timeline/utils/InsightAIContext.js",
   "front_end/panels/timeline/utils/SourceMapsResolver.js",
+  "front_end/panels/timeline/utils/Treemap.js",
   "front_end/panels/web_audio/AudioContextContentBuilder.js",
   "front_end/panels/web_audio/AudioContextSelector.js",
   "front_end/panels/web_audio/WebAudioModel.js",

front_end/models/trace/extras/ScriptDuplication.test.ts

@@ -29,6 +29,7 @@ async function loadScriptFixture(
     scriptId: `1.${name}` as Protocol.Runtime.ScriptId,
     frame: 'abcdef',
     ts: 0 as Trace.Types.Timing.Micro,
+    inline: false,
     content: fixture.content,
     sourceMap: new SDK.SourceMap.SourceMap(compiledUrl, mapUrl, fixture.sourceMapJson),
   };
@@ -38,8 +39,8 @@ describeWithEnvironment('ScriptDuplication', function() {
   describe('computeGeneratedFileSizes', () => {
     it('works (simple map)', async function() {
       const script = await loadScriptFixture('foo.min');
-      const results = Trace.Extras.ScriptDuplication.computeGeneratedFileSizes(script);
-      assert.deepEqual(results, {
+      const sizes = Trace.Handlers.ModelHandlers.Scripts.getScriptGeneratedSizes(script);
+      assert.deepEqual(sizes, {
         files: {
           'node_modules/browser-pack/_prelude.js': 480,
           'src/bar.js': 104,
@@ -52,8 +53,8 @@ describeWithEnvironment('ScriptDuplication', function() {
 
     it('works (complex map)', async function() {
       const script = await loadScriptFixture('squoosh');
-      const results = Trace.Extras.ScriptDuplication.computeGeneratedFileSizes(script);
-      assert.deepEqual(results, {
+      const sizes = Trace.Handlers.ModelHandlers.Scripts.getScriptGeneratedSizes(script);
+      assert.deepEqual(sizes, {
         files: {
           'webpack:///node_modules/comlink/comlink.js': 4117,
           'webpack:///node_modules/linkstate/dist/linkstate.es.js': 412,
@@ -140,8 +141,8 @@ describeWithEnvironment('ScriptDuplication', function() {
         // @ts-expect-error
         map.sources[1] = null;
       });
-      const results = Trace.Extras.ScriptDuplication.computeGeneratedFileSizes(script);
-      assert.deepEqual(results, {
+      const sizes = Trace.Handlers.ModelHandlers.Scripts.getScriptGeneratedSizes(script);
+      assert.deepEqual(sizes, {
         files: {
           'node_modules/browser-pack/_prelude.js': 480,
           null: 104,
@@ -156,8 +157,8 @@ describeWithEnvironment('ScriptDuplication', function() {
       const script = await loadScriptFixture('foo.min', fixture => {
         fixture.sourceMapJson.mappings = 'blahblah blah';
       });
-      const results = Trace.Extras.ScriptDuplication.computeGeneratedFileSizes(script);
-      assert.deepEqual(results, {
+      const sizes = Trace.Handlers.ModelHandlers.Scripts.getScriptGeneratedSizes(script);
+      assert.deepEqual(sizes, {
         files: {},
         totalBytes: 718,
         unmappedBytes: 718,
@@ -168,8 +169,8 @@ describeWithEnvironment('ScriptDuplication', function() {
       const script = await loadScriptFixture('foo.min', fixture => {
         fixture.content = 'blahblah blah';
       });
-      const results = Trace.Extras.ScriptDuplication.computeGeneratedFileSizes(script);
-      assert.deepEqual(results, {
+      const sizes = Trace.Handlers.ModelHandlers.Scripts.getScriptGeneratedSizes(script);
+      assert.deepEqual(sizes, {
         errorMessage: 'foo.min.js.map mapping for last column out of bounds: 1:14',
       });
     });
@@ -178,8 +179,8 @@ describeWithEnvironment('ScriptDuplication', function() {
       const script = await loadScriptFixture('foo.min', fixture => {
         fixture.content = '';
       });
-      const results = Trace.Extras.ScriptDuplication.computeGeneratedFileSizes(script);
-      assert.deepEqual(results, {
+      const sizes = Trace.Handlers.ModelHandlers.Scripts.getScriptGeneratedSizes(script);
+      assert.deepEqual(sizes, {
         errorMessage: 'foo.min.js.map mapping for column out of bounds: 1:1',
       });
     });
@@ -194,8 +195,8 @@ describeWithEnvironment('ScriptDuplication', function() {
             'AAAA';
         fixture.sourceMapJson.mappings = newMappings.join(',');
       });
-      const results = Trace.Extras.ScriptDuplication.computeGeneratedFileSizes(script);
-      assert.deepEqual(results, {
+      const sizes = Trace.Handlers.ModelHandlers.Scripts.getScriptGeneratedSizes(script);
+      assert.deepEqual(sizes, {
         errorMessage: 'foo.min.js.map mapping for last column out of bounds: 1:685',
       });
     });
@@ -208,8 +209,8 @@ describeWithEnvironment('ScriptDuplication', function() {
         // See https://sourcemaps.info/spec.html#:~:text=broken%20down%20as%20follows
         fixture.sourceMapJson.mappings = ';'.repeat(10) + fixture.sourceMapJson.mappings;
       });
-      const results = Trace.Extras.ScriptDuplication.computeGeneratedFileSizes(script);
-      assert.deepEqual(results, {
+      const sizes = Trace.Handlers.ModelHandlers.Scripts.getScriptGeneratedSizes(script);
+      assert.deepEqual(sizes, {
         errorMessage: 'foo.min.js.map mapping for line out of bounds: 11',
       });
     });
@@ -218,8 +219,8 @@ describeWithEnvironment('ScriptDuplication', function() {
       const script = await loadScriptFixture('foo.min', fixture => {
         fixture.sourceMapJson.names = ['blah'];
       });
-      const results = Trace.Extras.ScriptDuplication.computeGeneratedFileSizes(script);
-      assert.deepEqual(results, {
+      const sizes = Trace.Handlers.ModelHandlers.Scripts.getScriptGeneratedSizes(script);
+      assert.deepEqual(sizes, {
         files: {
           'node_modules/browser-pack/_prelude.js': 480,
           'src/bar.js': 104,
@@ -232,12 +233,17 @@ describeWithEnvironment('ScriptDuplication', function() {
   });
 
   describe('computeScriptDuplication', () => {
+    function getDuplication(scriptsData: Trace.Handlers.ModelHandlers.Scripts.ScriptsData):
+        Trace.Extras.ScriptDuplication.ScriptDuplication {
+      return Trace.Extras.ScriptDuplication.computeScriptDuplication(scriptsData).duplicationGroupedByNodeModules;
+    }
+
     it('works (simple, no duplication)', async () => {
       const scriptsData: Trace.Handlers.ModelHandlers.Scripts.ScriptsData = {
         scripts: [await loadScriptFixture('foo.min')],
       };
 
-      const results = Object.fromEntries(Trace.Extras.ScriptDuplication.computeScriptDuplication(scriptsData));
+      const results = Object.fromEntries(getDuplication(scriptsData));
       assert.deepEqual(results, {});
     });
 
@@ -246,8 +252,8 @@ describeWithEnvironment('ScriptDuplication', function() {
         scripts: [await loadScriptFixture('coursehero-bundle-1'), await loadScriptFixture('coursehero-bundle-2')],
       };
 
-      const results = Object.fromEntries(
-          [...Trace.Extras.ScriptDuplication.computeScriptDuplication(scriptsData).entries()].map(([key, data]) => {
+      const results =
+          Object.fromEntries([...getDuplication(scriptsData).entries()].map(([key, data]) => {
             return [
               key, data.duplicates.map(v => ({scriptId: v.script.scriptId as string, resourceSize: v.attributedSize}))
             ];

front_end/models/trace/extras/ScriptDuplication.ts

@@ -2,83 +2,13 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-import type * as SDK from '../../../core/sdk/sdk.js';
-import type * as Handlers from '../handlers/handlers.js';
+import * as Handlers from '../handlers/handlers.js';
 
 // Ignore modules smaller than an absolute threshold.
 const ABSOLUTE_SIZE_THRESHOLD_BYTES = 1024 * 0.5;
 // Ignore modules smaller than a % size of largest copy of the module.
 const RELATIVE_SIZE_THRESHOLD = 0.1;
 
-type GeneratedFileSizes = {
-  errorMessage: string,
-}|{files: Record<string, number>, unmappedBytes: number, totalBytes: number};
-
-/**
- * Using a script's contents and source map, attribute every generated byte to an authored source file.
- */
-export function computeGeneratedFileSizes(script: Handlers.ModelHandlers.Scripts.Script): GeneratedFileSizes {
-  if (!script.sourceMap) {
-    throw new Error('expected source map');
-  }
-
-  const map = script.sourceMap;
-  const content = script.content ?? '';
-  const contentLength = content.length;
-  const lines = content.split('\n');
-  const files: Record<string, number> = {};
-  const totalBytes = contentLength;
-  let unmappedBytes = totalBytes;
-
-  const lastGeneratedColumnMap = computeLastGeneratedColumnMap(script.sourceMap);
-
-  for (const mapping of map.mappings()) {
-    const source = mapping.sourceURL;
-    const lineNum = mapping.lineNumber;
-    const colNum = mapping.columnNumber;
-    const lastColNum = lastGeneratedColumnMap.get(mapping);
-
-    // Webpack sometimes emits null mappings.
-    // https://github.com/mozilla/source-map/pull/303
-    if (!source) {
-      continue;
-    }
-
-    // Lines and columns are zero-based indices. Visually, lines are shown as a 1-based index.
-
-    const line = lines[lineNum];
-    if (line === null || line === undefined) {
-      const errorMessage = `${map.url()} mapping for line out of bounds: ${lineNum + 1}`;
-      return {errorMessage};
-    }
-
-    if (colNum > line.length) {
-      const errorMessage = `${map.url()} mapping for column out of bounds: ${lineNum + 1}:${colNum}`;
-      return {errorMessage};
-    }
-
-    let mappingLength = 0;
-    if (lastColNum !== undefined) {
-      if (lastColNum > line.length) {
-        const errorMessage = `${map.url()} mapping for last column out of bounds: ${lineNum + 1}:${lastColNum}`;
-        return {errorMessage};
-      }
-      mappingLength = lastColNum - colNum;
-    } else {
-      // Add +1 to account for the newline.
-      mappingLength = line.length - colNum + 1;
-    }
-    files[source] = (files[source] || 0) + mappingLength;
-    unmappedBytes -= mappingLength;
-  }
-
-  return {
-    files,
-    unmappedBytes,
-    totalBytes,
-  };
-}
-
 interface SourceData {
   source: string;
   resourceSize: number;
@@ -195,7 +125,7 @@ export function getNodeModuleName(source: string): string {
 
 function groupByNodeModules(duplication: ScriptDuplication): ScriptDuplication {
   const groupedDuplication: ScriptDuplication = new Map();
-  for (const [source, data] of duplication.entries()) {
+  for (const [source, data] of duplication) {
     if (!source.includes('node_modules')) {
       groupedDuplication.set(source, data);
       continue;
@@ -222,38 +152,32 @@ function groupByNodeModules(duplication: ScriptDuplication): ScriptDuplication {
   return groupedDuplication;
 }
 
-function computeLastGeneratedColumnMap(map: SDK.SourceMap.SourceMap): Map<SDK.SourceMap.SourceMapEntry, number> {
-  const result = new Map<SDK.SourceMap.SourceMapEntry, number>();
-
-  const mappings = map.mappings();
-  for (let i = 0; i < mappings.length - 1; i++) {
-    const mapping = mappings[i];
-    const nextMapping = mappings[i + 1];
-    if (mapping.lineNumber === nextMapping.lineNumber) {
-      result.set(mapping, nextMapping.columnNumber);
-    }
-  }
-
-  // Now, all but the last mapping on each line will have 'lastColumnNumber' set to a number.
-  return result;
+/**
+ * Sort by estimated savings.
+ */
+function sorted(duplication: ScriptDuplication): ScriptDuplication {
+  return new Map([...duplication].sort((a, b) => b[1].estimatedDuplicateBytes - a[1].estimatedDuplicateBytes));
 }
 
 /**
- * Returns a @see ScriptDuplication for the given collection of script contents + source maps.
+ * Returns 2 @see ScriptDuplication for the given collection of script contents + source maps:
+ *
+ * 1. `duplication` keys correspond to authored files
+ * 2. `duplication` keys correspond to authored files, except all files within the same
+ *    node_module package are aggregated under the same entry.
  */
-export function computeScriptDuplication(scriptsData: Handlers.ModelHandlers.Scripts.ScriptsData): ScriptDuplication {
-  const sizesMap = new Map<Handlers.ModelHandlers.Scripts.Script, GeneratedFileSizes>();
-  for (const script of scriptsData.scripts) {
-    if (script.content && script.sourceMap) {
-      sizesMap.set(script, computeGeneratedFileSizes(script));
-    }
-  }
-
+export function computeScriptDuplication(scriptsData: Handlers.ModelHandlers.Scripts.ScriptsData):
+    {duplication: ScriptDuplication, duplicationGroupedByNodeModules: ScriptDuplication} {
   const sourceDatasMap = new Map<Handlers.ModelHandlers.Scripts.Script, SourceData[]>();
 
   // Determine size of each `sources` entry.
-  for (const [script, sizes] of sizesMap) {
-    if (!script.sourceMap) {
+  for (const script of scriptsData.scripts) {
+    if (!script.content || !script.sourceMap) {
+      continue;
+    }
+
+    const sizes = Handlers.ModelHandlers.Scripts.getScriptGeneratedSizes(script);
+    if (!sizes) {
       continue;
     }
 
@@ -279,7 +203,7 @@ export function computeScriptDuplication(scriptsData: Handlers.ModelHandlers.Scr
     }
   }
 
-  let duplication: ScriptDuplication = new Map();
+  const duplication: ScriptDuplication = new Map();
   for (const [script, sourceDataArray] of sourceDatasMap) {
     for (const sourceData of sourceDataArray) {
       let data = duplication.get(sourceData.source);
@@ -294,9 +218,13 @@ export function computeScriptDuplication(scriptsData: Handlers.ModelHandlers.Scr
     }
   }
 
-  duplication = groupByNodeModules(duplication);
+  const duplicationGroupedByNodeModules = groupByNodeModules(duplication);
+
   normalizeDuplication(duplication);
+  normalizeDuplication(duplicationGroupedByNodeModules);
 
-  // Sort by estimated savings.
-  return new Map([...duplication].sort((a, b) => b[1].estimatedDuplicateBytes - a[1].estimatedDuplicateBytes));
+  return {
+    duplication: sorted(duplication),
+    duplicationGroupedByNodeModules: sorted(duplicationGroupedByNodeModules),
+  };
 }

front_end/models/trace/handlers/ScriptsHandler.test.ts

@@ -42,6 +42,7 @@ describe('ScriptsHandler', () => {
         frame: '21D58E83A5C17916277166140F6A464B',
         request: undefined,
         ts: 50442438976,
+        inline: false,
         url: 'http://localhost:8080/index.html',
         content: 'source text 1',
         sourceMapUrl: 'http://localhost:8080/source.map.json',
@@ -53,6 +54,7 @@ describe('ScriptsHandler', () => {
         frame: '21D58E83A5C17916277166140F6A464B',
         request: undefined,
         ts: 50442438976,
+        inline: false,
         url: 'http://localhost:8080/index.html',
         content: 'source text 2'
       },
@@ -63,6 +65,7 @@ describe('ScriptsHandler', () => {
         content: ' text ',
         request: undefined,
         ts: 50442438976,
+        inline: false,
         url: 'http://localhost:8080/index.html',
       }
     ]);