More robust detection of private networks for high traffic websites (#178)

Author: philipp-classen

reporting/src/http.js

@@ -190,7 +190,7 @@ function headerOverrideViaWebRequestAPI({ url, headers, requestType }) {
       details.method !== 'GET' ||
       !matchesWebRequestApiType(requestType, details.type)
     ) {
-      // does that match the request that we intended to trigger
+      // does not match the request that we intended to trigger
       return {};
     }
 

reporting/src/network.js

@@ -16,6 +16,123 @@ const MINUTE = 60 * SECOND;
 const HOUR = 60 * MINUTE;
 const DAY = 24 * HOUR;
 
+// This list is not intended to be complete. For missing entries, the
+// DnsResolver class will fallback to dynamic mappings learned from
+// observing network requests (hostname -> IP).
+//
+// Why include explicit mappings at all? Because it provides stronger
+// guarantees during the bootstrapping phase. In a web extension context,
+// where no DNS resolution API is available, we otherwise could not guarantee
+// that the critical IP addresses would be always resolvable.
+const WELL_KNOWN_HOSTNAME_IS_PRIVATE_NETWORK = {
+  // list of private networks (completeness not required):
+  '127.0.0.1': true,
+  'localhost': true,
+  'fritz.box': true,
+
+  // (generated: top 100 public domains ranked by traffic):
+  'www.google.com': false,
+  'www.fiverr.com': false,
+  'www.youtube.com': false,
+  'www.amazon.com': false,
+  'www.reddit.com': false,
+  'www.facebook.com': false,
+  'mail.google.com': false,
+  'docs.google.com': false,
+  'x.com': false,
+  'accounts.google.com': false,
+  'www.instagram.com': false,
+  'www.bing.com': false,
+  'www.linkedin.com': false,
+  'login.microsoftonline.com': false,
+  'en.wikipedia.org': false,
+  'github.com': false,
+  'www.amazon.de': false,
+  'www.ebay.com': false,
+  'www.twitch.tv': false,
+  'www.amazon.co.jp': false,
+  'chatgpt.com': false,
+  'www.amazon.fr': false,
+  'www.msn.com': false,
+  'news.yahoo.co.jp': false,
+  'old.reddit.com': false,
+  'www.amazon.co.uk': false,
+  'www.roblox.com': false,
+  'drive.google.com': false,
+  'rule34.xxx': false,
+  'www.pornhub.com': false,
+  'www.imdb.com': false,
+  'xhamster.com': false,
+  'www.xvideos.com': false,
+  'e-hentai.org': false,
+  'www.paypal.com': false,
+  'www.amazon.ca': false,
+  'www.espn.com': false,
+  'www.bbc.co.uk': false,
+  'www.kleinanzeigen.de': false,
+  'www.nexusmods.com': false,
+  'steamcommunity.com': false,
+  'www.bilibili.com': false,
+  'www.ebay.co.uk': false,
+  'allegro.pl': false,
+  'meet.google.com': false,
+  'www.aliexpress.com': false,
+  'chaturbate.com': false,
+  'www.yahoo.co.jp': false,
+  'www.nytimes.com': false,
+  'www.canva.com': false,
+  'www.etsy.com': false,
+  'nhentai.net': false,
+  'www.ozon.ru': false,
+  'www.theguardian.com': false,
+  'www.amazon.it': false,
+  'hitomi.la': false,
+  'www.neopets.com': false,
+  'outlook.live.com': false,
+  'supjav.com': false,
+  'store.steampowered.com': false,
+  'www.deviantart.com': false,
+  'calendar.google.com': false,
+  'www.ecosia.org': false,
+  'mail.yahoo.com': false,
+  'de.fiverr.com': false,
+  'letterboxd.com': false,
+  'login.live.com': false,
+  'www.fmkorea.com': false,
+  'outlook.office.com': false,
+  'news.google.com': false,
+  'statics.teams.cdn.office.net': false,
+  'gall.dcinside.com': false,
+  'citizenfreepress.com': false,
+  'duckduckgo.com': false,
+  'www.amazon.es': false,
+  'www.xnxx.com': false,
+  'www.imagefap.com': false,
+  'www.pixiv.net': false,
+  'www.ikea.com': false,
+  'www.netflix.com': false,
+  'www.ebay.de': false,
+  'imgsrc.ru': false,
+  'www.dailymail.co.uk': false,
+  'www.marktplaats.nl': false,
+  'www.foxnews.com': false,
+  'auctions.yahoo.co.jp': false,
+  'www.booking.com': false,
+  'game.granbluefantasy.jp': false,
+  'www.erome.com': false,
+  'f95zone.to': false,
+  'www.cardmarket.com': false,
+  'www.chess.com': false,
+  'www.vinted.fr': false,
+  'www.discogs.com': false,
+  'www.ancestry.com': false,
+  'app.hubspot.com': false,
+  '1337x.to': false,
+  'www.upwork.com': false,
+  'search.yahoo.co.jp': false,
+  'www.patreon.com': false,
+};
+
 // Simple function to detect IP addresses that are non-public.
 // Local to the machine or link-only (belonging to a local network).
 //
@@ -61,14 +178,6 @@ export function isLocalIP(ip) {
   return false;
 }
 
-// There are two unsolved problems:
-// - What to do if there is no IP resolution cached yet?
-//   (Currently, it assumes that the page is private; but most of the
-//    time this assumption will be wrong, so it looks overly conservative.
-//    Perhaps, extending the API to return three values: yes/no/unknown
-//    could help. But it will push the complexity to the caller.)
-// - Perhaps adding all seen URLs from private IPs to the bloom filter
-//   could be a solution
 export class DnsResolver {
   constructor() {
     this.dns = new Map();
@@ -81,9 +190,22 @@ export class DnsResolver {
   }
 
   isPrivateHostname(hostname) {
-    if (hostname === 'localhost') {
-      return true;
+    // For a limited set of hostnames, we know how to classify them.
+    // By design, this will never cover all websites.
+    const isPrivate = WELL_KNOWN_HOSTNAME_IS_PRIVATE_NETWORK[hostname];
+    if (isPrivate !== undefined) {
+      return isPrivate;
     }
+
+    // For the long tail, fall back to checking the IP address. Since we lack a
+    // DNS resolution API, we use mappings built from observing network requests.
+    //
+    // Note: Since this is a heuristic, there remains the question on how to deal
+    // with hostnames that we have not seen before. That case should be rare
+    // enough, since we work with hostnames that originate from visited websites.
+    // Yet if no such mapping exists, we have to guess. Assuming a hostname is
+    // private by default would be overly conservative; instead, we default to
+    // public unless proven otherwise.
     const entry = this.dns.get(hostname);
     return entry?.ip && isLocalIP(entry.ip);
   }