[RPP] Save source maps to trace metadata for enhanced traces

Also modify source map fetching in RPP to 1) never do it for non-fresh
traces and 2) ensure that the raw source map is always available for
purposes of saving to trace metadata (required a hack in SourceMap to
not unconditionally drop the raw payload).

Bug: 394373632
Change-Id: Icff6d0da8d66111e19ba9fd966c6ab5ab014262e
Reviewed-on: https://chromium-review.googlesource.com/c/devtools/devtools-frontend/+/6287830
Reviewed-by: Paul Irish <[email protected]>
Commit-Queue: Connor Clark <[email protected]>
Auto-Submit: Connor Clark <[email protected]>

Author: Connor Clark

front_end/core/sdk/SourceMap.ts

@@ -148,6 +148,8 @@ interface SourceInfo {
 }
 
 export class SourceMap {
+  static retainRawSourceMaps = false;
+
   #json: SourceMapV3|null;
   readonly #compiledURLInternal: Platform.DevToolsPath.UrlString;
   readonly #sourceMappingURL: Platform.DevToolsPath.UrlString;
@@ -181,6 +183,10 @@ export class SourceMap {
     this.eachSection(this.parseSources.bind(this));
   }
 
+  json(): SourceMapV3|null {
+    return this.#json;
+  }
+
   augmentWithScopes(scriptUrl: Platform.DevToolsPath.UrlString, ranges: NamedFunctionRange[]): void {
     this.#ensureMappingsProcessed();
     if (this.#json && this.#json.version > 3) {
@@ -411,6 +417,9 @@ export class SourceMap {
       this.mappings().sort(SourceMapEntry.compare);
 
       this.#computeReverseMappings(this.#mappingsInternal);
+    }
+
+    if (!SourceMap.retainRawSourceMaps) {
       this.#json = null;
     }
   }

front_end/core/sdk/SourceMapManager.ts

@@ -182,6 +182,17 @@ export async function loadSourceMap(
   }
 }
 
+export async function tryLoadSourceMap(
+    url: Platform.DevToolsPath.UrlString, initiator: PageResourceLoadInitiator): Promise<SourceMapV3|null> {
+  try {
+    const {content} = await PageResourceLoader.instance().loadResource(url, initiator);
+    return parseSourceMap(content);
+  } catch (cause) {
+    console.error(`Could not load content for ${url}: ${cause.message}`, {cause});
+    return null;
+  }
+}
+
 interface ClientData {
   relativeSourceURL: Platform.DevToolsPath.UrlString;
   // Stores the raw sourceMappingURL as provided by V8. These are not guaranteed to

front_end/models/trace/handlers/ScriptsHandler.ts

@@ -124,6 +124,8 @@ export async function finalize(options: Types.Configuration.ParseOptions): Promi
       continue;
     }
 
+    script.sourceMapUrl = sourceMapUrl;
+
     const params: Types.Configuration.ResolveSourceMapParams = {
       scriptId: script.scriptId,
       scriptUrl: sourceUrl as Platform.DevToolsPath.UrlString,

front_end/models/trace/types/File.ts

@@ -192,6 +192,14 @@ export interface MetaData {
   enhancedTraceVersion?: number;
   modifications?: Modifications;
   cruxFieldData?: CrUXManager.PageResult[];
+  /** Currently only stores JS maps, not CSS. This never stores data url source maps. */
+  sourceMaps?: MetadataSourceMap[];
+}
+
+interface MetadataSourceMap {
+  url: string;
+  sourceMapUrl: string;
+  sourceMap: SDK.SourceMap.SourceMapV3;
 }
 
 export type Contents = TraceFile|Event[];

front_end/panels/timeline/TimelineController.ts

@@ -178,12 +178,22 @@ export class TimelineController implements Trace.TracingManager.TracingManagerCl
 
     this.client.loadingStarted();
 
-    const [fieldData] = await Promise.all([
-      this.fetchFieldData(),
-      // TODO(crbug.com/366072294): Report the progress of this resumption, as it can be lengthy on heavy pages.
-      SDK.TargetManager.TargetManager.instance().resumeAllTargets(),
-      this.waitForTracingToStop(),
-    ]);
+    // Give `TimelinePanel.#executeNewTrace` a chance to retain source maps from SDK.SourceMap.SourceMapManager.
+    SDK.SourceMap.SourceMap.retainRawSourceMaps = true;
+
+    const [fieldData] =
+        await Promise
+            .all([
+              this.fetchFieldData(),
+              // TODO(crbug.com/366072294): Report the progress of this resumption, as it can be lengthy on heavy pages.
+              SDK.TargetManager.TargetManager.instance().resumeAllTargets(),
+              this.waitForTracingToStop(),
+            ])
+            .catch(e => {
+              // Normally set false in allSourcesFinished, but just in case something fails, catch it here.
+              SDK.SourceMap.SourceMap.retainRawSourceMaps = false;
+              throw e;
+            });
     this.#fieldData = fieldData;
 
     // Now we re-enable throttling again to maintain the setting being persistent.
@@ -271,6 +281,7 @@ export class TimelineController implements Trace.TracingManager.TracingManagerCl
     const metadata = await this.createMetadata();
     await this.client.loadingComplete(this.#collectedEvents, /* exclusiveFilter= */ null, metadata);
     this.client.loadingCompleteForTest();
+    SDK.SourceMap.SourceMap.retainRawSourceMaps = false;
   }
 
   tracingBufferUsage(usage: number): void {

front_end/panels/timeline/TimelinePanel.ts

@@ -1400,7 +1400,10 @@ export class TimelinePanel extends UI.Panel.Panel implements Client, TimelineMod
           traceAsString = cpuprofileJsonGenerator(profile);
         }
       } else {
-        const formattedTraceIter = traceJsonGenerator(traceEvents, metadata);
+        const formattedTraceIter = traceJsonGenerator(traceEvents, {
+          ...metadata,
+          sourceMaps: isEnhancedTraces ? metadata?.sourceMaps : undefined,
+        });
         traceAsString = Array.from(formattedTraceIter).join('');
       }
       if (!traceAsString) {
@@ -2416,7 +2419,59 @@ export class TimelinePanel extends UI.Panel.Panel implements Client, TimelineMod
     });
   }
 
-  #createSourceMapResolver(isFreshRecording: boolean): Trace.TraceModel.ParseConfig['resolveSourceMap'] {
+  /**
+   * Store source maps on trace metadata (but just the non-data url ones).
+   *
+   * Many raw source maps are already in memory, but there are some cases where they may
+   * not be and have to be fetched here:
+   *
+   * 1. If the trace processor (via `#createSourceMapResolver`) never fetched it,
+   *    due to `ScriptHandler` skipping the script if it could not find an associated frame.
+   * 2. If the initial fetch failed (perhaps the failure was intermittent and a
+   *    subsequent attempt will work).
+   */
+  async #retainSourceMapsForEnhancedTrace(
+      parsedTrace: Trace.Handlers.Types.ParsedTrace, metadata: Trace.Types.File.MetaData): Promise<void> {
+    const handleScript = async(script: Trace.Handlers.ModelHandlers.Scripts.Script): Promise<void> => {
+      if (!script.sourceMapUrl || script.sourceMapUrl.startsWith('data:')) {
+        return;
+      }
+
+      if (metadata.sourceMaps?.find(m => m.sourceMapUrl === script.sourceMapUrl)) {
+        return;
+      }
+
+      // TimelineController sets `SDK.SourceMap.SourceMap.retainRawSourceMaps` to true,
+      // which means the raw source map is present (assuming `script.sourceMap` is too).
+      let rawSourceMap = script.sourceMap?.json();
+
+      // If the raw map is not present for some reason, fetch it again.
+      if (!rawSourceMap) {
+        const initiator = {
+          target: null,
+          frameId: script.frame as Protocol.Page.FrameId,
+          initiatorUrl: script.url as Platform.DevToolsPath.UrlString
+        };
+        rawSourceMap = await SDK.SourceMapManager.tryLoadSourceMap(
+            script.sourceMapUrl as Platform.DevToolsPath.UrlString, initiator);
+      }
+
+      if (script.url && rawSourceMap) {
+        metadata.sourceMaps?.push({url: script.url, sourceMapUrl: script.sourceMapUrl, sourceMap: rawSourceMap});
+      }
+    };
+
+    metadata.sourceMaps = [];
+
+    const promises = [];
+    for (const script of parsedTrace?.Scripts.scripts.values() ?? []) {
+      promises.push(handleScript(script));
+    }
+    await Promise.all(promises);
+  }
+
+  #createSourceMapResolver(isFreshRecording: boolean, metadata: Trace.Types.File.MetaData|null):
+      Trace.TraceModel.ParseConfig['resolveSourceMap'] {
     // Currently, only experimental insights need source maps.
     if (!Root.Runtime.experiments.isEnabled(Root.Runtime.ExperimentName.TIMELINE_EXPERIMENTAL_INSIGHTS)) {
       return;
@@ -2462,38 +2517,63 @@ export class TimelinePanel extends UI.Panel.Panel implements Client, TimelineMod
         }
       }
 
-      // Else... fetch it!
-      if (!scriptUrl) {
-        return null;
+      // If loading from disk, check the metadata for source maps.
+      // The metadata doesn't store data url source maps.
+      const isDataUrl = sourceMapUrl.startsWith('data:');
+      if (!isFreshRecording && metadata?.sourceMaps && !isDataUrl) {
+        const cachedSourceMap = metadata.sourceMaps.find(m => m.sourceMapUrl === sourceMapUrl);
+        if (cachedSourceMap) {
+          return new SDK.SourceMap.SourceMap(scriptUrl, sourceMapUrl, cachedSourceMap.sourceMap);
+        }
       }
 
-      // In all other cases, we must fetch the source map.
-      // For example, since the debugger model is disable during recording, any non-final navigations during
-      // the trace will never have their source maps fetched by the debugger model. That's only ever done here.
+      // Never fetch source maps if the trace is not fresh - the source maps may not
+      // reflect what was actually loaded by the page for this trace on disk.
+      if (!isFreshRecording && !isDataUrl) {
+        return null;
+      }
 
-      try {
-        const initiator = {target: null, frameId: frame, initiatorUrl: scriptUrl};
-        const payload = await SDK.SourceMapManager.loadSourceMap(sourceMapUrl, initiator);
-        return new SDK.SourceMap.SourceMap(scriptUrl, sourceMapUrl, payload);
-      } catch (cause) {
-        console.error(`Could not load content for ${sourceMapUrl}: ${cause.message}`, {cause});
+      if (!scriptUrl) {
+        return null;
       }
 
-      return null;
+      // In all other cases, fetch the source map.
+      //
+      // 1) data urls
+      // 2) fresh recording + source map not for active frame
+      //
+      // For example, since the debugger model is disable during recording, any
+      // non-final navigations during the trace will never have their source maps
+      // fetched by the debugger model. That's only ever done here.
+
+      const initiator = {target: null, frameId: frame, initiatorUrl: scriptUrl};
+      const payload = await SDK.SourceMapManager.tryLoadSourceMap(sourceMapUrl, initiator);
+      return payload ? new SDK.SourceMap.SourceMap(scriptUrl, sourceMapUrl, payload) : null;
     };
   }
 
   async #executeNewTrace(
       collectedEvents: Trace.Types.Events.Event[], isFreshRecording: boolean,
       metadata: Trace.Types.File.MetaData|null): Promise<void> {
-    return await this.#traceEngineModel.parse(
+    await this.#traceEngineModel.parse(
         collectedEvents,
         {
           metadata: metadata ?? undefined,
           isFreshRecording,
-          resolveSourceMap: this.#createSourceMapResolver(isFreshRecording),
+          resolveSourceMap: this.#createSourceMapResolver(isFreshRecording, metadata),
         },
     );
+
+    // Store all source maps on the trace metadata.
+    // If not fresh, we can't validate the maps are still accurate.
+    if (isFreshRecording && metadata &&
+        Root.Runtime.experiments.isEnabled(Root.Runtime.ExperimentName.TIMELINE_ENHANCED_TRACES)) {
+      const traceIndex = this.#traceEngineModel.lastTraceIndex();
+      const parsedTrace = this.#traceEngineModel.parsedTrace(traceIndex);
+      if (parsedTrace) {
+        await this.#retainSourceMapsForEnhancedTrace(parsedTrace, metadata);
+      }
+    }
   }
 
   loadingCompleteForTest(): void {