[RPP] Set up 3p cache

This starts building the caches from the NetworkHandler down to the
RendererHandler. Also moves URLForEntry so that it could be depended on
by the Handlers.

This is not yet used anywhere, but this sets the data that we need
to be referenced in follow up features like the checkbox, summary
details table, etc.

This also doesn't yet handle keeping the cache up to date via
sourceMaps, this will be worked through on a separate cl

Bug:381960419
Change-Id: I9629f5a511908166ad104aad37b2590c04b0a23f
Reviewed-on: https://chromium-review.googlesource.com/c/devtools/devtools-frontend/+/6069068
Reviewed-by: Adam Raine <[email protected]>
Commit-Queue: Adriana Ixba <[email protected]>

Author: Adriana Ixba

config/gni/devtools_grd_files.gni

@@ -1051,7 +1051,6 @@ grd_files_debug_sources = [
   "front_end/models/trace/extras/TimelineJSProfile.js",
   "front_end/models/trace/extras/TraceFilter.js",
   "front_end/models/trace/extras/TraceTree.js",
-  "front_end/models/trace/extras/URLForEntry.js",
   "front_end/models/trace/handlers/AnimationFramesHandler.js",
   "front_end/models/trace/handlers/AnimationHandler.js",
   "front_end/models/trace/handlers/AsyncJSCallsHandler.js",
@@ -1083,6 +1082,7 @@ grd_files_debug_sources = [
   "front_end/models/trace/handlers/UserTimingsHandler.js",
   "front_end/models/trace/handlers/WarningsHandler.js",
   "front_end/models/trace/handlers/WorkersHandler.js",
+  "front_end/models/trace/handlers/helpers.js",
   "front_end/models/trace/handlers/types.js",
   "front_end/models/trace/helpers/Extensions.js",
   "front_end/models/trace/helpers/Network.js",
@@ -1897,6 +1897,7 @@ grd_files_debug_sources = [
   "front_end/panels/timeline/timelinePanel.css.js",
   "front_end/panels/timeline/timelineStatusDialog.css.js",
   "front_end/panels/timeline/utils/AICallTree.js",
+  "front_end/panels/timeline/utils/EntityMapper.js",
   "front_end/panels/timeline/utils/EntryName.js",
   "front_end/panels/timeline/utils/EntryStyles.js",
   "front_end/panels/timeline/utils/Helpers.js",

front_end/models/trace/Processor.test.ts

@@ -118,6 +118,7 @@ describeWithEnvironment('TraceProcessor', function() {
           Renderer: Trace.Handlers.ModelHandlers.Renderer,
           Samples: Trace.Handlers.ModelHandlers.Samples,
           AuctionWorklets: Trace.Handlers.ModelHandlers.AuctionWorklets,
+          NetworkRequests: Trace.Handlers.ModelHandlers.NetworkRequests,
         },
         Trace.Types.Configuration.defaults());
 

front_end/models/trace/extras/BUILD.gn

@@ -18,7 +18,6 @@ devtools_module("extras") {
     "TimelineJSProfile.ts",
     "TraceFilter.ts",
     "TraceTree.ts",
-    "URLForEntry.ts",
   ]
 
   deps = [
@@ -53,7 +52,6 @@ ts_library("unittests") {
     "ThirdParties.test.ts",
     "TraceFilter.test.ts",
     "TraceTree.test.ts",
-    "URLForEntry.test.ts",
   ]
 
   deps = [

front_end/models/trace/extras/ThirdParties.test.ts

@@ -8,33 +8,6 @@ import * as Trace from '../trace.js';
 
 describeWithEnvironment('ThirdParties', function() {
   describe('Entities', function() {
-    it('correctly makes up entities', async function() {
-      const expectedEntities = new Map<string, string>([
-        ['http://localhost:8080/', 'localhost'],
-        ['https://fonts.googleapis.com/css2?family=Orelega+One&display=swap', 'googleapis.com'],
-        ['https://emp.bbci.co.uk/emp/bump-4/bump-4.js', 'bbci.co.uk'],
-        ['http://localhost:8080/blocking.js', 'localhost'],
-        ['https://fonts.gstatic.com/s/orelegaone/v1/3qTpojOggD2XtAdFb-QXZFt93kY.woff2', 'gstatic.com'],
-        ['chrome-extension://chromeextension/something/exciting.js', 'chromeextension'],
-      ]);
-
-      for (const [url, expectedEntity] of expectedEntities.entries()) {
-        const gotEntity =
-            Trace.Extras.ThirdParties.makeUpEntity(new Map<string, Trace.Extras.ThirdParties.Entity>(), url)?.name ??
-            '';
-        assert.deepEqual(gotEntity, expectedEntity);
-      }
-    });
-    it('coreectly makes up chrome extension entity', async function() {
-      const url = 'chrome-extension://chromeextension/something/exciting.js';
-      const gotEntity =
-          Trace.Extras.ThirdParties.makeUpEntity(new Map<string, Trace.Extras.ThirdParties.Entity>(), url);
-      assert.exists(gotEntity);
-
-      assert.deepEqual(gotEntity.name, 'chromeextension');
-      assert.deepEqual(gotEntity.category, 'Chrome Extension');
-      assert.deepEqual(gotEntity.homepage, 'https://chromewebstore.google.com/detail/chromeextension');
-    });
     it('gets correct entitiesByRequest', async function() {
       const {parsedTrace} = await TraceLoader.traceEngine(this, 'load-simple.json.gz');
       const reqs = parsedTrace.NetworkRequests.byTime;

front_end/models/trace/extras/ThirdParties.ts

@@ -3,12 +3,10 @@
 // found in the LICENSE file.
 
 import * as ThirdPartyWeb from '../../../third_party/third-party-web/third-party-web.js';
-import type * as Handlers from '../handlers/handlers.js';
+import * as Handlers from '../handlers/handlers.js';
 import * as Helpers from '../helpers/helpers.js';
 import * as Types from '../types/types.js';
 
-import * as URLForEntry from './URLForEntry.js';
-
 export type Entity = typeof ThirdPartyWeb.ThirdPartyWeb.entities[number];
 
 export interface Summary {
@@ -22,77 +20,6 @@ export interface SummaryMaps {
   requestsByEntity: Map<Entity, Types.Events.SyntheticNetworkRequest[]>;
 }
 
-/**
- * Returns the origin portion of a Chrome extension URL.
- */
-function getChromeExtensionOrigin(url: URL): string {
-  return url.protocol + '//' + url.host;
-}
-
-function makeUpChromeExtensionEntity(entityCache: Map<string, Entity>, url: string, extensionName?: string): Entity {
-  const parsedUrl = new URL(url);
-  const origin = getChromeExtensionOrigin(parsedUrl);
-  const host = new URL(origin).host;
-  const name = extensionName || host;
-
-  const cachedEntity = entityCache.get(origin);
-  if (cachedEntity) {
-    return cachedEntity;
-  }
-
-  const chromeExtensionEntity = {
-    name,
-    company: name,
-    category: 'Chrome Extension',
-    homepage: 'https://chromewebstore.google.com/detail/' + host,
-    categories: [],
-    domains: [],
-    averageExecutionTime: 0,
-    totalExecutionTime: 0,
-    totalOccurrences: 0,
-  };
-
-  entityCache.set(origin, chromeExtensionEntity);
-  return chromeExtensionEntity;
-}
-
-export function makeUpEntity(entityCache: Map<string, Entity>, url: string): Entity|undefined {
-  if (url.startsWith('chrome-extension:')) {
-    return makeUpChromeExtensionEntity(entityCache, url);
-  }
-
-  // Make up an entity only for valid http/https URLs.
-  if (!url.startsWith('http')) {
-    return;
-  }
-
-  // NOTE: Lighthouse uses a tld database to determine the root domain, but here
-  // we are using third party web's database. Doesn't really work for the case of classifying
-  // domains 3pweb doesn't know about, so it will just give us a guess.
-  const rootDomain = ThirdPartyWeb.ThirdPartyWeb.getRootDomain(url);
-  if (!rootDomain) {
-    return;
-  }
-
-  if (entityCache.has(rootDomain)) {
-    return entityCache.get(rootDomain);
-  }
-
-  const unrecognizedEntity = {
-    name: rootDomain,
-    company: rootDomain,
-    category: '',
-    categories: [],
-    domains: [rootDomain],
-    averageExecutionTime: 0,
-    totalExecutionTime: 0,
-    totalOccurrences: 0,
-    isUnrecognized: true,
-  };
-  entityCache.set(rootDomain, unrecognizedEntity);
-  return unrecognizedEntity;
-}
-
 function getSelfTimeByUrl(
     parsedTrace: Handlers.Types.ParsedTrace, bounds: Types.Timing.TraceWindowMicroSeconds): Map<string, number> {
   const selfTimeByUrl = new Map<string, number>();
@@ -118,7 +45,7 @@ function getSelfTimeByUrl(
             continue;
           }
 
-          const url = URLForEntry.getNonResolved(parsedTrace as Handlers.Types.ParsedTrace, event);
+          const url = Handlers.Helpers.getNonResolvedURL(event, parsedTrace as Handlers.Types.ParsedTrace);
           if (!url) {
             continue;
           }
@@ -138,7 +65,7 @@ export function getEntitiesByRequest(requests: Types.Events.SyntheticNetworkRequ
   const madeUpEntityCache = new Map<string, Entity>();
   for (const request of requests) {
     const url = request.args.data.url;
-    const entity = ThirdPartyWeb.ThirdPartyWeb.getEntity(url) ?? makeUpEntity(madeUpEntityCache, url);
+    const entity = ThirdPartyWeb.ThirdPartyWeb.getEntity(url) ?? Handlers.Helpers.makeUpEntity(madeUpEntityCache, url);
     if (entity) {
       entityByRequest.set(request, entity);
     }

front_end/models/trace/extras/URLForEntry.ts

@@ -11,4 +11,3 @@ export * as ThirdParties from './ThirdParties.js';
 export * as TimelineJSProfile from './TimelineJSProfile.js';
 export * as TraceFilter from './TraceFilter.js';
 export * as TraceTree from './TraceTree.js';
-export * as URLForEntry from './URLForEntry.js';

front_end/models/trace/extras/extras.ts

@@ -40,12 +40,14 @@ devtools_module("handlers") {
     "UserTimingsHandler.ts",
     "WarningsHandler.ts",
     "WorkersHandler.ts",
+    "helpers.ts",
     "types.ts",
   ]
 
   deps = [
     "../../../core/platform:bundle",
     "../../../generated",
+    "../../../third_party/third-party-web:bundle",
     "../../cpu_profile:bundle",
     "../helpers:bundle",
     "../types:bundle",
@@ -97,6 +99,7 @@ ts_library("unittests") {
     "UserTimingsHandler.test.ts",
     "WarningsHandler.test.ts",
     "WorkersHandler.test.ts",
+    "helpers.test.ts",
   ]
 
   deps = [

front_end/models/trace/handlers/BUILD.gn

@@ -342,6 +342,112 @@ describe('NetworkRequestsHandler', function() {
       assert.strictEqual(initiator.args.data.url, event.args.data.stackTrace?.[0].url);
     });
   });
+  describe('ThirdParty caches', () => {
+    it('Correctly captures entities by network event', async function() {
+      const traceEvents = await TraceLoader.rawEvents(this, 'lantern/paul/trace.json.gz');
+      for (const event of traceEvents) {
+        Trace.Handlers.ModelHandlers.Meta.handleEvent(event);
+        Trace.Handlers.ModelHandlers.NetworkRequests.handleEvent(event);
+      }
+      await Trace.Handlers.ModelHandlers.Meta.finalize();
+      await Trace.Handlers.ModelHandlers.NetworkRequests.finalize();
+
+      const {entityMappings} = Trace.Handlers.ModelHandlers.NetworkRequests.data();
+      const syntheticNetworkEventsByEntity = new Map(
+          Array.from(entityMappings.eventsByEntity.entries()).map(([entity, events]) => {
+            const syntheticNetworkEvents = events.filter(
+                event => Trace.Types.Events.isSyntheticNetworkRequest(event),
+            );
+            return [entity, syntheticNetworkEvents];
+          }),
+      );
+      const requestsByEntityResult = [...syntheticNetworkEventsByEntity.entries()].map(([entity, requests]) => {
+        return [entity.name, requests.map(r => r.args?.data?.url)];
+      });
+      assert.deepEqual(
+          requestsByEntityResult,
+          [
+            [
+              'paulirish.com',
+              [
+                'https://www.paulirish.com/',
+                'https://www.paulirish.com/assets/wikipedia-flamechart.jpg',
+                'https://www.paulirish.com/avatar150.jpg',
+                'https://www.paulirish.com/javascripts/modernizr-2.0.js',
+                'https://www.paulirish.com/javascripts/ender.js',
+                'https://www.paulirish.com/javascripts/octopress.js',
+                'https://www.paulirish.com/javascripts/firebase-performance-standalone.js',
+                'https://www.paulirish.com/images/noise.png?1418840251',
+                'https://www.paulirish.com/images/code_bg.png?1418840251',
+                'https://www.paulirish.com/favicon.ico',
+              ],
+            ],
+            [
+              'Google Tag Manager',
+              [
+                'https://www.googletagmanager.com/gtag/js?id=G-PGXNGYWP8E',
+              ],
+            ],
+            [
+              'Google Fonts',
+              [
+                'https://fonts.googleapis.com/css?family=PT+Serif:regular,italic,bold|PT+Sans:regular,italic,bold|Droid+Sans:400,700|Lato:700,900',
+                'https://fonts.gstatic.com/s/droidsans/v18/SlGVmQWMvZQIdix7AFxXkHNSbRYXags.woff2',
+                'https://fonts.gstatic.com/s/lato/v24/S6u9w4BMUTPHh6UVSwiPGQ3q5d0.woff2',
+                'https://fonts.gstatic.com/s/ptsans/v17/jizaRExUiTo99u79D0KExcOPIDU.woff2',
+                'https://fonts.gstatic.com/s/ptsans/v17/jizfRExUiTo99u79B_mh0O6tLR8a8zI.woff2',
+                'https://fonts.gstatic.com/s/droidsans/v18/SlGWmQWMvZQIdix7AFxXmMh3eDs1ZyHKpWg.woff2',
+                'https://fonts.gstatic.com/s/ptserif/v18/EJRVQgYoZZY2vCFuvAFWzr-_dSb_.woff2',
+              ],
+            ],
+            [
+              'Google Analytics',
+              [
+                'https://www.google-analytics.com/analytics.js',
+                'https://www.google-analytics.com/g/collect?v=2&tid=G-PGXNGYWP8E&gtm=45je4580v880158425za200&_p=1715625261583&gcd=13l3l3l3l1&npa=0&dma=0&cid=414801335.1715625262&ul=en-us&sr=412x823&uaa=&uab=64&uafvl=Not%252FA)Brand%3B8.0.0.0%7CChromium%3B126.0.6475.0%7CGoogle%2520Chrome%3B126.0.6475.0&uamb=1&uam=moto%20g%20power%20(2022)&uap=Android&uapv=11.0&uaw=0&are=1&frm=0&pscdl=noapi&_s=1&sid=1715625261&sct=1&seg=0&dl=https%3A%2F%2Fwww.paulirish.com%2F&dt=Paul%20Irish&en=page_view&_fv=1&_nsi=1&_ss=1&_ee=1&tfd=353',
+                'https://www.google-analytics.com/j/collect?v=1&_v=j101&a=272264939&t=pageview&_s=1&dl=https%3A%2F%2Fwww.paulirish.com%2F&ul=en-us&de=UTF-8&dt=Paul%20Irish&sd=30-bit&sr=412x823&vp=412x823&je=0&_u=IADAAEABAAAAACAAI~&jid=1388679807&gjid=654531532&cid=414801335.1715625262&tid=UA-692547-2&_gid=1964734610.1715625262&_r=1&_slc=1&z=1746264594',
+              ],
+            ],
+            [
+              'Disqus',
+              [
+                'https://paulirish.disqus.com/count.js',
+              ],
+            ],
+            [
+              'Firebase',
+              [
+                'https://firebaseinstallations.googleapis.com/v1/projects/paulirishcom/installations',
+                'https://firebaseremoteconfig.googleapis.com/v1/projects/paulirishcom/namespaces/fireperf:fetch?key=AIzaSyCGxLbbFQxH4BV1fY0RODlxTos9nJa2l_g',
+              ],
+            ],
+          ],
+      );
+    });
+    it('Correctly captures entities', async function() {
+      const traceEvents = await TraceLoader.rawEvents(this, 'lantern/paul/trace.json.gz');
+      for (const event of traceEvents) {
+        Trace.Handlers.ModelHandlers.Meta.handleEvent(event);
+        Trace.Handlers.ModelHandlers.NetworkRequests.handleEvent(event);
+      }
+      await Trace.Handlers.ModelHandlers.Meta.finalize();
+      await Trace.Handlers.ModelHandlers.NetworkRequests.finalize();
+
+      const {entityMappings} = Trace.Handlers.ModelHandlers.NetworkRequests.data();
+      const expectedEntities = [
+        'paulirish.com',
+        'Google Tag Manager',
+        'Google Fonts',
+        'Google Analytics',
+        'Disqus',
+        'Firebase',
+      ];
+      const gotEntities = Array.from(entityMappings.entityByEvent.values()).map(enity => enity.name);
+      expectedEntities.forEach(entity => {
+        assert.isTrue(gotEntities.includes(entity));
+      });
+    });
+  });
 });
 
 function assertDataArgsStats<D extends keyof DataArgs>(