Update third-party-web to 0.26.2

Connor Clark · Devtools-frontend LUCI CQ · commit c653121b9563 · 2025-02-04T10:54:44.000-08:00
Bug: None Change-Id: Idd491df20d8692c0943e336052e27c826d9401e1 Reviewed-on: https://chromium-review.googlesource.com/c/devtools/devtools-frontend/+/6221254 Reviewed-by: Adriana Ixba <aixba@chromium.org> Commit-Queue: Adriana Ixba <aixba@chromium.org> Auto-Submit: Connor Clark <cjamcl@chromium.org>
diff --git a/front_end/third_party/third-party-web/README.chromium b/front_end/third_party/third-party-web/README.chromium
@@ -1,8 +1,8 @@
 Name: third-party-web
 Short Name: third-party-web
 URL: https://github.com/patrickhulce/third-party-web
-Version: 0.24.5
-Revision: 82f19004382b7b41b2cb41d8a81cb982a6ffe622
+Version: 0.26.2
+Revision: d60863a2ea913ed26ae0446296a0cb1cc6c71291
 License: MIT
 License File: LICENSE
 Security Critical: no
diff --git a/front_end/third_party/third-party-web/lib/nostats-subset.js b/front_end/third_party/third-party-web/lib/nostats-subset.js
diff --git a/front_end/third_party/third-party-web/package.json b/front_end/third_party/third-party-web/package.json
@@ -3,6 +3,7 @@
   "description": "Package list used when building/upgrading third-party-web",
   "private": true,
   "dependencies": {
-    "esbuild": "0.23.1"
+    "esbuild": "0.23.1",
+    "third-party-web": "0.26.2"
   }
 }
diff --git a/front_end/third_party/third-party-web/package/.prettierrc b/front_end/third_party/third-party-web/package/.prettierrc
diff --git a/front_end/third_party/third-party-web/package/README.md b/front_end/third_party/third-party-web/package/README.md
diff --git a/front_end/third_party/third-party-web/package/dist/entities-httparchive-nostats.json b/front_end/third_party/third-party-web/package/dist/entities-httparchive-nostats.json
diff --git a/front_end/third_party/third-party-web/package/dist/entities-httparchive.json b/front_end/third_party/third-party-web/package/dist/entities-httparchive.json
diff --git a/front_end/third_party/third-party-web/package/dist/entities-nostats.json b/front_end/third_party/third-party-web/package/dist/entities-nostats.json
diff --git a/front_end/third_party/third-party-web/package/dist/entities.json b/front_end/third_party/third-party-web/package/dist/entities.json
diff --git a/front_end/third_party/third-party-web/package/facades.md b/front_end/third_party/third-party-web/package/facades.md
@@ -0,0 +1,46 @@
+# Facade data in third-party-web
+
+In addition to identifying entities, third-party-web also includes data on entities' _products_, and available alternative libraries for products known as _facades_. A **third-party product** is a third-party resource that is included on a page associated with a specific integration/SDK/API, for example: a YouTube embed iframe. A **facade** is a frontend element which looks similar to the actual product and lazily loads it upon user interaction.  It's expected to be significantly more lightweight/faster. For example: [lite-youtube-embed](https://github.com/paulirish/lite-youtube-embed).
+
+[Lighthouse](https://github.com/GoogleChrome/lighthouse) 7.0 has a new audit that [identifies third-party embeds](https://web.dev/third-party-facades/) that can be lazy loaded with a facade. The Lighthouse audit is powered by the facade data in third-party-web.
+
+## Criteria for adding a new facade
+
+### Basic functionality
+
+* Loads a frontend "component" which looks like the actual third-party embed.
+* After some user interaction (click, mouseover, etc), component replaces itself with the actual third-party embed.
+
+### Well Maintained
+
+* The projects issues and contributions are managed responsibly. Bugs are 
+  handled swiftly (e.g within 60 days of filing).
+* Is already used in production, ideally by users other than the creator.
+
+### Orthogonal
+
+* The facade handles a use case not covered by existing facades. For example: a 
+  lite-youtube clone isn't differentiated, but a react port of lite-youtube 
+  would work.
+
+### Small Payload
+
+* Should be significantly smaller in comparison to the deferred third-party 
+  product.
+    * The JS/CSS/image/etc payload should be the minimum size necessary to mimic 
+      the product. (A rough rule of thumb is a facade should be &lt; 10% of the 
+      product it's mimicking)
+    * We'd expect the facade to score 99+ in Lighthouse if it's tested on an 
+      otherwise-blank page.
+    * Any required images should be as small as is reasonable. Using blurred, low-res,
+      or [SQIP](https://calendar.perfplanet.com/2017/sqip-vague-vectors-for-performant-previews/) 
+      techniques may be advantageous.
+
+## Submission Process
+
+Anyone can propose changes, by making a pull request to this repo. Proposals 
+don't necessarily need to be submitted by the creator.
+
+Read 
+[contributing](https://github.com/patrickhulce/third-party-web#contributing) for 
+basics; look in `data/entities.js` for `products` to see how the data is structured.
diff --git a/front_end/third_party/third-party-web/package/lib/__snapshots__/index.test.js.snap b/front_end/third_party/third-party-web/package/lib/__snapshots__/index.test.js.snap
@@ -977,7 +977,7 @@ Array [
   "undefined - https://cloud.roistat.com/static/marketplace/Viber/script.js",
   "ShareThis - https://count-server.sharethis.com/v2.0/get_counts",
   "undefined - https://m.fg8dgt.com/ul_cb/sync",
-  "Crazy Egg - https://my.hellobar.com/scripts.hellobar.comad1844ba7b83c9bf3f0241424beb70be4c4252f1.js",
+  "Hello Bar - https://my.hellobar.com/scripts.hellobar.comad1844ba7b83c9bf3f0241424beb70be4c4252f1.js",
   "PrintFriendly - https://cdn.printfriendly.com/rails-public-legacy/images/pf-icon.gif",
   "undefined - wss://visitors.live/socket.io/",
   "undefined - wss://in.visitors.live/socket.io/",
diff --git a/front_end/third_party/third-party-web/package/lib/index.test.js b/front_end/third_party/third-party-web/package/lib/index.test.js
@@ -59,7 +59,7 @@ describe('getEntity', () => {
   it('works for direct domain usage', () => {
     expect(getEntity('https://js.connect.facebook.net/lib.js')).toMatchInlineSnapshot(`
       Object {
-        "averageExecutionTime": 387.5557268806942,
+        "averageExecutionTime": 347.4278160199557,
         "categories": Array [
           "social",
         ],
@@ -103,16 +103,16 @@ describe('getEntity', () => {
             ],
           },
         ],
-        "totalExecutionTime": 1232848097,
-        "totalOccurrences": 3181086,
+        "totalExecutionTime": 1097107210,
+        "totalOccurrences": 3157799,
       }
     `)
   })
 
   it('works for inferred domain usage', () => {
     expect(getEntity('https://unknown.typekit.net/fonts.css')).toMatchInlineSnapshot(`
       Object {
-        "averageExecutionTime": 339.5325083117843,
+        "averageExecutionTime": 660.2645605704683,
         "categories": Array [
           "cdn",
         ],
@@ -129,8 +129,8 @@ describe('getEntity', () => {
         "homepage": "https://fonts.adobe.com/",
         "name": "Adobe TypeKit",
         "products": Array [],
-        "totalExecutionTime": 34926351,
-        "totalOccurrences": 102866,
+        "totalExecutionTime": 78981507,
+        "totalOccurrences": 119621,
       }
     `)
   })
diff --git a/front_end/third_party/third-party-web/package/lib/markdown/faqs.partial.md b/front_end/third_party/third-party-web/package/lib/markdown/faqs.partial.md
@@ -0,0 +1,36 @@
+---
+name: faqs
+---
+
+### I don't see entity X in the list. What's up with that?
+
+This can be for one of several reasons:
+
+1.  The entity does not have references to their origin on at least 50 pages in the dataset.
+1.  The entity's origins have not yet been identified. See [How can I contribute?](#contribute)
+
+### What is "Total Occurences"?
+
+Total Occurrences is the number of pages on which the entity is included.
+
+### How is the "Average Impact" determined?
+
+The HTTP Archive dataset includes Lighthouse reports for each URL on mobile. Lighthouse has an audit called "bootup-time" that summarizes the amount of time that each script spent on the main thread. The "Average Impact" for an entity is the total execution time of scripts whose domain matches one of the entity's domains divided by the total number of pages that included the entity.
+
+```
+Average Impact = Total Execution Time / Total Occurrences
+```
+
+### How does Lighthouse determine the execution time of each script?
+
+Lighthouse's bootup time audit attempts to attribute all toplevel main-thread tasks to a URL. A main thread task is attributed to the first script URL found in the stack. If you're interested in helping us improve this logic, see [Contributing](#contributing) for details.
+
+### The data for entity X seems wrong. How can it be corrected?
+
+Verify that the origins in `data/entities.js` are correct. Most issues will simply be the result of mislabelling of shared origins. If everything checks out, there is likely no further action and the data is valid. If you still believe there's errors, file an issue to discuss futher.
+
+<a name="contribute"></a>
+
+### How can I contribute?
+
+Only about 90% of the third party script execution has been assigned to an entity. We could use your help identifying the rest! See [Contributing](#contributing) for details.
diff --git a/front_end/third_party/third-party-web/package/lib/markdown/goals.partial.md b/front_end/third_party/third-party-web/package/lib/markdown/goals.partial.md
@@ -0,0 +1,9 @@
+---
+name: goals
+---
+
+1.  Quantify the impact of third party scripts on the web.
+1.  Identify the third party scripts on the web that have the greatest performance cost.
+1.  Give developers the information they need to make informed decisions about which third parties to include on their sites.
+1.  Incentivize responsible third party script behavior.
+1.  Make this information accessible and useful.
diff --git a/front_end/third_party/third-party-web/package/lib/markdown/methodology.partial.md b/front_end/third_party/third-party-web/package/lib/markdown/methodology.partial.md
@@ -0,0 +1,5 @@
+---
+name: methodology
+---
+
+[HTTP Archive](https://httparchive.org/) is an initiative that tracks how the web is built. Every month, ~4 million sites are crawled with [Lighthouse](https://github.com/GoogleChrome/lighthouse) on mobile. Lighthouse breaks down the total script execution time of each page and attributes the execution to a URL. Using [BigQuery](https://cloud.google.com/bigquery/), this project aggregates the script execution to the origin-level and assigns each origin to the responsible entity.
diff --git a/front_end/third_party/third-party-web/package/lib/markdown/template.md b/front_end/third_party/third-party-web/package/lib/markdown/template.md
@@ -0,0 +1,151 @@
+# [Third Party Web](https://www.thirdpartyweb.today/)
+
+## Check out the shiny new web UI https://www.thirdpartyweb.today/
+
+Data on third party entities and their impact on the web.
+
+This document is a summary of which third party scripts are most responsible for excessive JavaScript execution on the web today.
+
+## Table of Contents
+
+1.  [Goals](#goals)
+1.  [Methodology](#methodology)
+1.  [npm Module](#npm-module)
+1.  [Updates](#updates)
+1.  [Data](#data)
+    1.  [Summary](#summary)
+    1.  [How to Interpret](#how-to-interpret)
+    1.  [Third Parties by Category](#by-category)
+        <%= category_table_of_contents %>
+    1.  [Third Parties by Total Impact](#by-total-impact)
+1.  [Future Work](#future-work)
+1.  [FAQs](#faqs)
+1.  [Contributing](#contributing)
+
+## Goals
+
+<%= partials.goals %>
+
+## Methodology
+
+<%= partials.methodology %>
+
+## npm Module
+
+The entity classification data is available as an npm module.
+
+```js
+const {getEntity} = require('third-party-web')
+const entity = getEntity('https://d36mpcpuzc4ztk.cloudfront.net/js/visitor.js')
+console.log(entity)
+//   {
+//     "name": "Freshdesk",
+//     "homepage": "https://freshdesk.com/",
+//     "category": "customer-success",
+//     "domains": ["d36mpcpuzc4ztk.cloudfront.net"]
+//   }
+```
+
+## Updates
+
+<%= updates_contents %>
+
+## Data
+
+### Summary
+
+Across top ~4 million sites, ~2700 origins account for ~57% of all script execution time with the top 50 entities already accounting for ~47%. Third party script execution is the majority chunk of the web today, and it's important to make informed choices.
+
+### How to Interpret
+
+Each entity has a number of data points available.
+
+1.  **Usage (Total Number of Occurrences)** - how many scripts from their origins were included on pages
+1.  **Total Impact (Total Execution Time)** - how many seconds were spent executing their scripts across the web
+1.  **Average Impact (Average Execution Time)** - on average, how many milliseconds were spent executing each script
+1.  **Category** - what type of script is this
+
+<a name="by-category"></a>
+
+### Third Parties by Category
+
+This section breaks down third parties by category. The third parties in each category are ranked from first to last based on the average impact of their scripts. Perhaps the most important comparisons lie here. You always need to pick an analytics provider, but at least you can pick the most well-behaved analytics provider.
+
+#### Overall Breakdown
+
+Unsurprisingly, ads account for the largest identifiable chunk of third party script execution.
+
+![breakdown by category](./by-category.png)
+
+<%= category_contents %>
+
+<a name="by-total-impact"></a>
+
+### Third Parties by Total Impact
+
+This section highlights the entities responsible for the most script execution across the web. This helps inform which improvements would have the largest total impact.
+
+<%= all_data %>
+
+## Future Work
+
+1.  Introduce URL-level data for more fine-grained analysis, i.e. which libraries from Cloudflare/Google CDNs are most expensive.
+1.  Expand the scope, i.e. include more third parties and have greater entity/category coverage.
+
+## FAQs
+
+<%= partials.faqs %>
+
+## Contributing
+
+### Thanks
+
+A **huge** thanks to [@simonhearne](https://twitter.com/simonhearne) and [@soulgalore](https://twitter.com/soulislove) for their assistance in classifying additional domains!
+
+### Updating the Entities
+
+The domain->entity mapping can be found in `data/entities.js`. Adding a new entity is as simple as adding a new array item with the following form.
+
+```js
+{
+    "name": "Facebook",
+    "homepage": "https://www.facebook.com",
+    "category": "social",
+    "domains": [
+        "*.facebook.com",
+        "*.fbcdn.net"
+    ],
+    "examples": [
+        "www.facebook.com",
+        "connect.facebook.net",
+        "staticxx.facebook.com",
+        "static.xx.fbcdn.net",
+        "m.facebook.com"
+    ]
+}
+```
+
+### Updating Attribution Logic
+
+The logic for attribution to individual script URLs can be found in the [Lighthouse repo](https://github.com/GoogleChrome/lighthouse). File an issue over there to discuss further.
+
+### Updating the Data
+
+This is now automated! Run `yarn start:update-ha-data` with a `gcp-credentials.json` file in the root directory of this project (look at `bin/automated-update.js` for the steps involved).
+
+### Updating this README
+
+This README is auto-generated from the templates `lib/` and the computed data. In order to update the charts, you'll need to make sure you have `cairo` installed locally in addition to `yarn install`.
+
+```bash
+# Install `cairo` and dependencies for node-canvas
+brew install pkg-config cairo pango libpng jpeg giflib
+# Build the requirements in this repo
+yarn build
+# Regenerate the README
+yarn start
+```
+
+### Updating the website
+
+The web code is located in `www/` directory of this repository. Open a PR to make changes.
diff --git a/front_end/third_party/third-party-web/package/lib/markdown/updates/2019-02-01.md b/front_end/third_party/third-party-web/package/lib/markdown/updates/2019-02-01.md
@@ -0,0 +1 @@
+Huge props to [WordAds](https://wordads.co/) for reducing their impact from ~2.5s to ~200ms on average! A few entities are showing considerably less data this cycle (Media Math, Crazy Egg, DoubleVerify, Bootstrap CDN). Perhaps they've added new CDNs/hostnames that we haven't identified or the basket of sites in HTTPArchive has shifted away from their usage.
diff --git a/front_end/third_party/third-party-web/package/lib/markdown/updates/2019-03-01.md b/front_end/third_party/third-party-web/package/lib/markdown/updates/2019-03-01.md
@@ -0,0 +1 @@
+Almost 2,000 entities tracked now across ~3,000+ domains! Huge props to [@simonhearne](https://twitter.com/simonhearne) for making this massive increase possible. Tag Managers have now been split out into their own category since they represented such a large percentage of the "Mixed / Other" category.
diff --git a/front_end/third_party/third-party-web/package/lib/markdown/updates/2019-05-06.md b/front_end/third_party/third-party-web/package/lib/markdown/updates/2019-05-06.md
@@ -0,0 +1 @@
+Google Ads clarified that `www.googletagservices.com` serves more ad scripts than generic tag management, and it has been reclassified accordingly. This has dropped the overall Tag Management share considerably back down to its earlier position.
diff --git a/front_end/third_party/third-party-web/package/lib/markdown/updates/2019-05-13.md b/front_end/third_party/third-party-web/package/lib/markdown/updates/2019-05-13.md
@@ -0,0 +1,14 @@
+A shortcoming of the attribution approach has been fixed. Total usage is now reported based on the number of _pages_ in the dataset that use the third-party, not the number of _scripts_. Correspondingly, all average impact times are now reported _per page_ rather than _per script_. Previously, a third party could appear to have a lower impact or be more popular simply by splitting their work across multiple files.
+
+Third-parties that performed most of their work from a single script should see little to no impact from this change, but some entities have seen significant ranking movement. Hosting providers that host entire pages are, understandably, the most affected.
+
+Some notable changes below:
+
+| Third-Party | Previously (per-script) | Now (per-page) |
+| ----------- | ----------------------- | -------------- |
+| Beeketing   | 137 ms                  | 465 ms         |
+| Sumo        | 263 ms                  | 798 ms         |
+| Tumblr      | 324 ms                  | 1499 ms        |
+| Yandex APIs | 393 ms                  | 1231 ms        |
+| Google Ads  | 402 ms                  | 1285 ms        |
+| Wix         | 972 ms                  | 5393 ms        |
diff --git a/front_end/third_party/third-party-web/package/lib/markdown/updates/2021-01-01.md b/front_end/third_party/third-party-web/package/lib/markdown/updates/2021-01-01.md
@@ -0,0 +1 @@
+Due to a change in HTTPArchive measurement which temporarily disabled site-isolation (out-of-process iframes), all of the third-parties whose work previously took place off the main-thread are now counted _on_ the main thread (and thus appear in our stats). This is most evident in the change to Google-owned properties such as YouTube and Doubleclick whose _complete_ cost are now captured.
diff --git a/front_end/third_party/third-party-web/package/lib/markdown/updates/2024-07-01.md b/front_end/third_party/third-party-web/package/lib/markdown/updates/2024-07-01.md
@@ -0,0 +1,3 @@
+Some third parties use a dynamic subdomain to serve its main script on websites (e.g .domain.com). Some of these subdomain scripts are saved under observed-domains JSON file as results of the `sql/all-observed-domains-query.sql` query but analyzing http archive database we found a lot that are ignored because of number of occurrences (less than 50 ).
+
+So, we've created a new query to keep observed domains with occurrence below 50 only if its mapped entity (based on entity.js) has a total occurrence (of all its declared domain) greater than 50.
diff --git a/front_end/third_party/third-party-web/package/nostats-subset.d.ts b/front_end/third_party/third-party-web/package/nostats-subset.d.ts
@@ -0,0 +1 @@
+export * from './lib/subsets/nostats.js'
diff --git a/front_end/third_party/third-party-web/package/package.json b/front_end/third_party/third-party-web/package/package.json
@@ -1,6 +1,6 @@
 {
   "name": "third-party-web",
-  "version": "0.24.5",
+  "version": "0.26.2",
   "description": "Categorized data on third party entities on the web.",
   "main": "./lib/index.js",
   "scripts": {
@@ -24,7 +24,7 @@
   },
   "license": "MIT",
   "devDependencies": {
-    "@google-cloud/bigquery": "^5.10.0",
+    "@google-cloud/bigquery": "^7.9.0",
     "chart.js": "^2.9.4",
     "chartjs-node-canvas": "^3.2.0",
     "colors": "^1.4.0",
@@ -33,13 +33,14 @@
     "lodash": "^4.17.15",
     "prettier": "^1.18.2",
     "prompts": "^2.4.2",
-    "semantic-release": "^19.0.2"
+    "semantic-release": "^24.2.0"
   },
   "config": {
     "exportAliases": {
       "nostats-subset": "./lib/subsets/nostats.js",
       "httparchive-nostats-subset": "./lib/subsets/httparchive-nostats.js",
       "httparchive-subset": "./lib/subsets/httparchive.js"
     }
-  }
+  },
+  "packageManager": "yarn@4.1.1"
 }
diff --git a/front_end/third_party/third-party-web/rebuild.sh b/front_end/third_party/third-party-web/rebuild.sh

Original file line number	Diff line number	Diff line change
`@@ -3,6 +3,7 @@`
`3`	`3`	`"description": "Package list used when building/upgrading third-party-web",`
`4`	`4`	`"private": true,`
`5`	`5`	`"dependencies": {`
`6`		`- "esbuild": "0.23.1"`
	`6`	`+ "esbuild": "0.23.1",`
	`7`	`+ "third-party-web": "0.26.2"`
`7`	`8`	`}`
`8`	`9`	`}`