[RPP]: Move third parties summaries extraction to helper

This also gets third party summaries based on a tracebounds

Bug:372881026,374192107
Change-Id: I6c2c965aea67033ee190bd8f55a25bb2e084f42c
Reviewed-on: https://chromium-review.googlesource.com/c/devtools/devtools-frontend/+/5939714
Commit-Queue: Adriana Ixba <[email protected]>
Reviewed-by: Connor Clark <[email protected]>

Author: Adriana Ixba

config/gni/devtools_grd_files.gni

@@ -1028,6 +1028,7 @@ grd_files_debug_sources = [
   "front_end/models/trace/extras/FilmStrip.js",
   "front_end/models/trace/extras/MainThreadActivity.js",
   "front_end/models/trace/extras/Metadata.js",
+  "front_end/models/trace/extras/ThirdParties.js",
   "front_end/models/trace/extras/URLForEntry.js",
   "front_end/models/trace/handlers/AnimationHandler.js",
   "front_end/models/trace/handlers/AuctionWorkletsHandler.js",

front_end/models/trace/extras/BUILD.gn

@@ -13,6 +13,7 @@ devtools_module("extras") {
     "FilmStrip.ts",
     "MainThreadActivity.ts",
     "Metadata.ts",
+    "ThirdParties.ts",
     "URLForEntry.ts",
   ]
 
@@ -44,6 +45,7 @@ ts_library("unittests") {
     "FilmStrip.test.ts",
     "MainThreadActivity.test.ts",
     "Metadata.test.ts",
+    "ThirdParties.test.ts",
     "URLForEntry.test.ts",
   ]
 

front_end/models/trace/extras/ThirdParties.test.ts

@@ -0,0 +1,117 @@
+// Copyright 2024 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import {describeWithEnvironment} from '../../../testing/EnvironmentHelpers.js';
+import {TraceLoader} from '../../../testing/TraceLoader.js';
+import * as Trace from '../trace.js';
+
+describeWithEnvironment('ThirdParties', function() {
+  describe('Entities', function() {
+    it('correctly makes up entities', async function() {
+      const expectedEntities = new Map<string, string>([
+        ['http://localhost:8080/', 'localhost'],
+        ['https://fonts.googleapis.com/css2?family=Orelega+One&display=swap', 'googleapis.com'],
+        ['https://emp.bbci.co.uk/emp/bump-4/bump-4.js', 'bbci.co.uk'],
+        ['http://localhost:8080/blocking.js', 'localhost'],
+        ['https://fonts.gstatic.com/s/orelegaone/v1/3qTpojOggD2XtAdFb-QXZFt93kY.woff2', 'gstatic.com'],
+        ['chrome-extension://chromeextension/something/exciting.js', 'chromeextension'],
+      ]);
+
+      for (const [url, expectedEntity] of expectedEntities.entries()) {
+        const gotEntity =
+            Trace.Extras.ThirdParties.makeUpEntity(new Map<string, Trace.Extras.ThirdParties.Entity>(), url)?.name ??
+            '';
+        assert.deepEqual(gotEntity, expectedEntity);
+      }
+    });
+    it('coreectly makes up chrome extension entity', async function() {
+      const url = 'chrome-extension://chromeextension/something/exciting.js';
+      const gotEntity =
+          Trace.Extras.ThirdParties.makeUpEntity(new Map<string, Trace.Extras.ThirdParties.Entity>(), url);
+      assert.exists(gotEntity);
+
+      assert.deepEqual(gotEntity.name, 'chromeextension');
+      assert.deepEqual(gotEntity.category, 'Chrome Extension');
+      assert.deepEqual(gotEntity.homepage, 'https://chromewebstore.google.com/detail/chromeextension');
+    });
+    it('gets correct entitiesByRequest', async function() {
+      const {parsedTrace} = await TraceLoader.traceEngine(this, 'load-simple.json.gz');
+      const reqs = parsedTrace.NetworkRequests.byTime;
+
+      const got = Trace.Extras.ThirdParties.getEntitiesByRequest(reqs);
+      const gotEntityByRequest = [...got.entityByRequest.entries()].map(([req, entity]) => {
+        return [req.args.data.url, entity.name];
+      });
+
+      assert.deepEqual(gotEntityByRequest, [
+        ['http://localhost:8080/', 'localhost'],
+        ['https://fonts.googleapis.com/css2?family=Orelega+One&display=swap', 'Google Fonts'],
+        ['http://localhost:8080/styles.css', 'localhost'],
+        ['http://localhost:8080/blocking.js', 'localhost'],
+        ['http://localhost:8080/module.js', 'localhost'],
+        ['https://fonts.gstatic.com/s/orelegaone/v1/3qTpojOggD2XtAdFb-QXZFt93kY.woff2', 'Google Fonts'],
+      ]);
+    });
+  });
+  describe('byTraceBounds', function() {
+    it('full trace bounds', async function() {
+      const {parsedTrace} = await TraceLoader.traceEngine(this, 'load-simple.json.gz');
+      const reqs = parsedTrace.NetworkRequests.byTime;
+
+      const {entityByRequest} = Trace.Extras.ThirdParties.getSummariesAndEntitiesForTraceBounds(
+          parsedTrace, parsedTrace.Meta.traceBounds, reqs);
+
+      const gotEntityByRequest = [...entityByRequest.entries()].map(([req, entity]) => {
+        return [req.args.data.url, entity.name];
+      });
+
+      assert.deepEqual(gotEntityByRequest, [
+        ['http://localhost:8080/', 'localhost'],
+        ['https://fonts.googleapis.com/css2?family=Orelega+One&display=swap', 'Google Fonts'],
+        ['http://localhost:8080/styles.css', 'localhost'],
+        ['http://localhost:8080/blocking.js', 'localhost'],
+        ['http://localhost:8080/module.js', 'localhost'],
+        ['https://fonts.gstatic.com/s/orelegaone/v1/3qTpojOggD2XtAdFb-QXZFt93kY.woff2', 'Google Fonts'],
+      ]);
+    });
+    it('partial trace bounds', async function() {
+      const {parsedTrace} = await TraceLoader.traceEngine(this, 'load-simple.json.gz');
+      const reqs = parsedTrace.NetworkRequests.byTime;
+
+      // Font requests of load-simple.json.gz begin & end before/after this bounds.
+      const min = Trace.Types.Timing.MicroSeconds(1634222300000);
+      const max = Trace.Types.Timing.MicroSeconds(1634222320000);
+      const middle = {min, max, range: Trace.Types.Timing.MicroSeconds(max - min)};
+
+      const {entityByRequest} =
+          Trace.Extras.ThirdParties.getSummariesAndEntitiesForTraceBounds(parsedTrace, middle, reqs);
+      const gotEntityByRequest = [...entityByRequest.entries()].map(([req, entity]) => {
+        return [req.args.data.url, entity.name];
+      });
+
+      // Only these localhost requests overlap traceBounds.
+      assert.deepEqual(gotEntityByRequest, [
+        ['http://localhost:8080/', 'localhost'],
+        ['http://localhost:8080/styles.css', 'localhost'],
+        ['http://localhost:8080/blocking.js', 'localhost'],
+        ['http://localhost:8080/module.js', 'localhost'],
+      ]);
+    });
+    it('no requests within trace bounds', async function() {
+      const {parsedTrace} = await TraceLoader.traceEngine(this, 'load-simple.json.gz');
+      const reqs = parsedTrace.NetworkRequests.byTime;
+
+      const min = Trace.Types.Timing.MicroSeconds(1634230000000);
+      const max = Trace.Types.Timing.MicroSeconds(1634231000000);
+      const middle = {min, max, range: Trace.Types.Timing.MicroSeconds(max - min)};
+
+      const {entityByRequest} =
+          Trace.Extras.ThirdParties.getSummariesAndEntitiesForTraceBounds(parsedTrace, middle, reqs);
+      const gotEntityByRequest = [...entityByRequest.entries()].map(([req, entity]) => {
+        return [req.args.data.url, entity.name];
+      });
+      assert.deepEqual(gotEntityByRequest, []);
+    });
+  });
+});

front_end/models/trace/extras/ThirdParties.ts

@@ -0,0 +1,207 @@
+// Copyright 2024 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import * as ThirdPartyWeb from '../../../third_party/third-party-web/third-party-web.js';
+import type * as Handlers from '../handlers/handlers.js';
+import * as Helpers from '../helpers/helpers.js';
+import * as Types from '../types/types.js';
+
+import * as URLForEntry from './URLForEntry.js';
+
+export type Entity = typeof ThirdPartyWeb.ThirdPartyWeb.entities[number];
+
+export interface Summary {
+  transferSize: number;
+  mainThreadTime: Types.Timing.MicroSeconds;
+}
+
+export interface SummaryMaps {
+  byEntity: Map<Entity, Summary>;
+  byRequest: Map<Types.Events.SyntheticNetworkRequest, Summary>;
+  requestsByEntity: Map<Entity, Types.Events.SyntheticNetworkRequest[]>;
+}
+
+/**
+ * Returns the origin portion of a Chrome extension URL.
+ */
+function getChromeExtensionOrigin(url: URL): string {
+  return url.protocol + '//' + url.host;
+}
+
+function makeUpChromeExtensionEntity(entityCache: Map<string, Entity>, url: string, extensionName?: string): Entity {
+  const parsedUrl = new URL(url);
+  const origin = getChromeExtensionOrigin(parsedUrl);
+  const host = new URL(origin).host;
+  const name = extensionName || host;
+
+  const cachedEntity = entityCache.get(origin);
+  if (cachedEntity) {
+    return cachedEntity;
+  }
+
+  const chromeExtensionEntity = {
+    name,
+    company: name,
+    category: 'Chrome Extension',
+    homepage: 'https://chromewebstore.google.com/detail/' + host,
+    categories: [],
+    domains: [],
+    averageExecutionTime: 0,
+    totalExecutionTime: 0,
+    totalOccurrences: 0,
+  };
+
+  entityCache.set(origin, chromeExtensionEntity);
+  return chromeExtensionEntity;
+}
+
+export function makeUpEntity(entityCache: Map<string, Entity>, url: string): Entity|undefined {
+  if (url.startsWith('chrome-extension:')) {
+    return makeUpChromeExtensionEntity(entityCache, url);
+  }
+
+  // Make up an entity only for valid http/https URLs.
+  if (!url.startsWith('http')) {
+    return;
+  }
+
+  // NOTE: Lighthouse uses a tld database to determine the root domain, but here
+  // we are using third party web's database. Doesn't really work for the case of classifying
+  // domains 3pweb doesn't know about, so it will just give us a guess.
+  const rootDomain = ThirdPartyWeb.ThirdPartyWeb.getRootDomain(url);
+  if (!rootDomain) {
+    return;
+  }
+
+  if (entityCache.has(rootDomain)) {
+    return entityCache.get(rootDomain);
+  }
+
+  const unrecognizedEntity = {
+    name: rootDomain,
+    company: rootDomain,
+    category: '',
+    categories: [],
+    domains: [rootDomain],
+    averageExecutionTime: 0,
+    totalExecutionTime: 0,
+    totalOccurrences: 0,
+    isUnrecognized: true,
+  };
+  entityCache.set(rootDomain, unrecognizedEntity);
+  return unrecognizedEntity;
+}
+
+function getSelfTimeByUrl(
+    parsedTrace: Handlers.Types.ParsedTrace, bounds: Types.Timing.TraceWindowMicroSeconds): Map<string, number> {
+  const selfTimeByUrl = new Map<string, number>();
+
+  for (const process of parsedTrace.Renderer.processes.values()) {
+    if (!process.isOnMainFrame) {
+      continue;
+    }
+
+    for (const thread of process.threads.values()) {
+      if (thread.name === 'CrRendererMain') {
+        if (!thread.tree) {
+          break;
+        }
+
+        for (const event of thread.entries) {
+          if (!Helpers.Timing.eventIsInBounds(event, bounds)) {
+            continue;
+          }
+
+          const node = parsedTrace.Renderer.entryToNode.get(event);
+          if (!node || !node.selfTime) {
+            continue;
+          }
+
+          const url = URLForEntry.getNonResolved(parsedTrace as Handlers.Types.ParsedTrace, event);
+          if (!url) {
+            continue;
+          }
+
+          selfTimeByUrl.set(url, node.selfTime + (selfTimeByUrl.get(url) ?? 0));
+        }
+      }
+    }
+  }
+
+  return selfTimeByUrl;
+}
+
+export function getEntitiesByRequest(requests: Types.Events.SyntheticNetworkRequest[]):
+    {entityByRequest: Map<Types.Events.SyntheticNetworkRequest, Entity>, madeUpEntityCache: Map<string, Entity>} {
+  const entityByRequest = new Map<Types.Events.SyntheticNetworkRequest, Entity>();
+  const madeUpEntityCache = new Map<string, Entity>();
+  for (const request of requests) {
+    const url = request.args.data.url;
+    const entity = ThirdPartyWeb.ThirdPartyWeb.getEntity(url) ?? makeUpEntity(madeUpEntityCache, url);
+    if (entity) {
+      entityByRequest.set(request, entity);
+    }
+  }
+  return {entityByRequest, madeUpEntityCache};
+}
+
+function getSummaryMap(
+    requests: Types.Events.SyntheticNetworkRequest[],
+    entityByRequest: Map<Types.Events.SyntheticNetworkRequest, Entity>,
+    selfTimeByUrl: Map<string, number>): SummaryMaps {
+  const byRequest = new Map<Types.Events.SyntheticNetworkRequest, Summary>();
+  const byEntity = new Map<Entity, Summary>();
+  const defaultSummary: Summary = {transferSize: 0, mainThreadTime: Types.Timing.MicroSeconds(0)};
+
+  for (const request of requests) {
+    const urlSummary = byRequest.get(request) || {...defaultSummary};
+    urlSummary.transferSize += request.args.data.encodedDataLength;
+    urlSummary.mainThreadTime =
+        Types.Timing.MicroSeconds(urlSummary.mainThreadTime + (selfTimeByUrl.get(request.args.data.url) ?? 0));
+    byRequest.set(request, urlSummary);
+  }
+
+  // Map each request's stat to a particular entity.
+  const requestsByEntity = new Map<Entity, Types.Events.SyntheticNetworkRequest[]>();
+  for (const [request, requestSummary] of byRequest.entries()) {
+    const entity = entityByRequest.get(request);
+    if (!entity) {
+      byRequest.delete(request);
+      continue;
+    }
+
+    const entitySummary = byEntity.get(entity) || {...defaultSummary};
+    entitySummary.transferSize += requestSummary.transferSize;
+    entitySummary.mainThreadTime =
+        Types.Timing.MicroSeconds(entitySummary.mainThreadTime + requestSummary.mainThreadTime);
+    byEntity.set(entity, entitySummary);
+
+    const entityRequests = requestsByEntity.get(entity) || [];
+    entityRequests.push(request);
+    requestsByEntity.set(entity, entityRequests);
+  }
+
+  return {byEntity, byRequest, requestsByEntity};
+}
+
+export function getSummariesAndEntitiesForTraceBounds(
+    parsedTrace: Handlers.Types.ParsedTrace, traceBounds: Types.Timing.TraceWindowMicroSeconds,
+    networkRequests: Types.Events.SyntheticNetworkRequest[]): {
+  summaries: SummaryMaps,
+  entityByRequest: Map<Types.Events.SyntheticNetworkRequest, Entity>,
+  madeUpEntityCache: Map<string, Entity>,
+} {
+  // Ensure we only handle requests that are within the given traceBounds.
+  const reqs = networkRequests.filter(event => {
+    return Helpers.Timing.eventIsInBounds(event, traceBounds);
+  });
+
+  const {entityByRequest, madeUpEntityCache} = getEntitiesByRequest(reqs);
+
+  const selfTimeByUrl = getSelfTimeByUrl(parsedTrace, traceBounds);
+  // TODO(crbug.com/352244718): re-work to still collect main thread activity if no request is present
+  const summaries = getSummaryMap(reqs, entityByRequest, selfTimeByUrl);
+
+  return {summaries, entityByRequest, madeUpEntityCache};
+}

front_end/models/trace/extras/extras.ts

@@ -6,4 +6,5 @@ export * as FetchNodes from './FetchNodes.js';
 export * as FilmStrip from './FilmStrip.js';
 export * as MainThreadActivity from './MainThreadActivity.js';
 export * as Metadata from './Metadata.js';
+export * as ThirdParties from './ThirdParties.js';
 export * as URLForEntry from './URLForEntry.js';

front_end/models/trace/helpers/BUILD.gn

@@ -23,6 +23,7 @@ devtools_module("helpers") {
     "../../../core/platform:bundle",
     "../../../core/root:bundle",
     "../../../models/cpu_profile:bundle",
+    "../../../third_party/third-party-web:bundle",
     "../types:bundle",
   ]
 }