Prepare bot spoofing protection

Author: timokoessler

library/agent/Agent.ts

@@ -378,11 +378,16 @@ export class Agent {
     }
 
     try {
-      const { blockedIPAddresses, blockedUserAgents, allowedIPAddresses } =
-        await fetchBlockedLists(this.token);
+      const {
+        blockedIPAddresses,
+        blockedUserAgents,
+        allowedIPAddresses,
+        botSpoofingData,
+      } = await fetchBlockedLists(this.token);
       this.serviceConfig.updateBlockedIPAddresses(blockedIPAddresses);
       this.serviceConfig.updateBlockedUserAgents(blockedUserAgents);
       this.serviceConfig.updateAllowedIPAddresses(allowedIPAddresses);
+      this.serviceConfig.updateBotSpoofingData(botSpoofingData);
     } catch (error: any) {
       console.error(`Aikido: Failed to update blocked lists: ${error.message}`);
     }

library/agent/ServiceConfig.ts

@@ -2,7 +2,14 @@ import { IPMatcher } from "../helpers/ip-matcher/IPMatcher";
 import { LimitedContext, matchEndpoints } from "../helpers/matchEndpoints";
 import { isPrivateIP } from "../vulnerabilities/ssrf/isPrivateIP";
 import type { Endpoint, EndpointConfig } from "./Config";
-import { IPList } from "./api/fetchBlockedLists";
+import { BotSpoofingData, IPList } from "./api/fetchBlockedLists";
+
+export type ServiceConfigBotSpoofingData = {
+  key: string;
+  uaPattern: RegExp;
+  ips: IPMatcher | undefined;
+  hostnames: string[];
+};
 
 export class ServiceConfig {
   private blockedUserIds: Map<string, string> = new Map();
@@ -19,6 +26,7 @@ export class ServiceConfig {
     allowlist: IPMatcher;
     description: string;
   }[] = [];
+  private botSpoofingData: ServiceConfigBotSpoofingData[] = [];
 
   constructor(
     endpoints: EndpointConfig[],
@@ -208,4 +216,26 @@ export class ServiceConfig {
   hasReceivedAnyStats() {
     return this.receivedAnyStats;
   }
+
+  updateBotSpoofingData(data: BotSpoofingData[]) {
+    this.botSpoofingData = [];
+
+    for (const source of data) {
+      // Skip empty
+      if (source.ips.length === 0 && source.hostnames.length === 0) {
+        continue;
+      }
+
+      this.botSpoofingData.push({
+        key: source.key,
+        uaPattern: new RegExp(source.uaPattern, "i"),
+        ips: new IPMatcher(source.ips),
+        hostnames: source.hostnames,
+      });
+    }
+  }
+
+  getBotSpoofingData() {
+    return this.botSpoofingData;
+  }
 }

library/agent/api/fetchBlockedLists.ts

@@ -8,10 +8,18 @@ export type IPList = {
   ips: string[];
 };
 
+export type BotSpoofingData = {
+  key: string;
+  uaPattern: string;
+  ips: string[];
+  hostnames: string[];
+};
+
 export async function fetchBlockedLists(token: Token): Promise<{
   blockedIPAddresses: IPList[];
   allowedIPAddresses: IPList[];
   blockedUserAgents: string;
+  botSpoofingData: BotSpoofingData[];
 }> {
   const baseUrl = getAPIURL();
   const { body, statusCode } = await fetch({
@@ -38,6 +46,7 @@ export async function fetchBlockedLists(token: Token): Promise<{
     blockedIPAddresses: IPList[];
     allowedIPAddresses: IPList[];
     blockedUserAgents: string;
+    botSpoofingData: BotSpoofingData[];
   } = JSON.parse(body);
 
   return {
@@ -54,5 +63,9 @@ export async function fetchBlockedLists(token: Token): Promise<{
       result && typeof result.blockedUserAgents === "string"
         ? result.blockedUserAgents
         : "",
+    botSpoofingData:
+      result && Array.isArray(result.botSpoofingData)
+        ? result.botSpoofingData
+        : [],
   };
 }

library/vulnerabilities/bot-spoofing/checkRequestForBotSpoofing.ts

@@ -0,0 +1,34 @@
+import type { Agent } from "../../agent/Agent";
+import type { Context } from "../../agent/Context";
+import { verifyBotAuthenticity } from "./verifyBotAuthenticity";
+
+export function checkRequestForBotSpoofing(context: Context, agent: Agent) {
+  const botSpoofingData = agent.getConfig().getBotSpoofingData();
+
+  if (!botSpoofingData || botSpoofingData.length === 0) {
+    return false;
+  }
+
+  const userAgent = context.headers["user-agent"];
+  const ip = context.remoteAddress;
+
+  if (!ip) {
+    return false;
+  }
+
+  if (typeof userAgent !== "string" || userAgent.length === 0) {
+    return false;
+  }
+
+  // Check if the user agent matches any of the bot spoofing patterns
+  const matchingBot = botSpoofingData.find((data) =>
+    data.uaPattern.test(userAgent)
+  );
+
+  if (!matchingBot) {
+    // The request is not from a protected bot
+    return false;
+  }
+
+  return verifyBotAuthenticity(ip, matchingBot);
+}

library/vulnerabilities/bot-spoofing/verifyBotAuthenticity.test.ts

@@ -0,0 +1,41 @@
+import * as t from "tap";
+import { verifyBotAuthenticity } from "./verifyBotAuthenticity";
+import { IPMatcher } from "../../helpers/ip-matcher/IPMatcher";
+
+t.test("it works with a matching IP", async (t) => {
+  const matchingBot = {
+    key: "bot",
+    uaPattern: /bot/i,
+    ips: new IPMatcher(["123.123.0.0/16"]),
+    hostnames: [],
+  };
+
+  t.same(await verifyBotAuthenticity("", matchingBot), false);
+  t.same(await verifyBotAuthenticity("1.2.3.4", matchingBot), false);
+  t.same(await verifyBotAuthenticity("123.123.1.2", matchingBot), true);
+  t.same(await verifyBotAuthenticity("123.123.123.123", matchingBot), true);
+});
+
+t.test("it works with hostnames (googlebot)", async (t) => {
+  const matchingBot = {
+    key: "google_test",
+    uaPattern: /Googlebot/i,
+    ips: new IPMatcher(),
+    hostnames: ["google.com", "googlebot.com"],
+  };
+
+  t.same(await verifyBotAuthenticity("1.1.1.1", matchingBot), false);
+  t.same(await verifyBotAuthenticity("66.249.90.77", matchingBot), true);
+});
+
+t.test("it works with hostnames (bingbot)", async (t) => {
+  const matchingBot = {
+    key: "bing_test",
+    uaPattern: /207.46.13.14/i,
+    ips: new IPMatcher(),
+    hostnames: ["search.msn.com", "bing.com"],
+  };
+
+  t.same(await verifyBotAuthenticity("1.1.1.1", matchingBot), false);
+  t.same(await verifyBotAuthenticity("207.46.13.14", matchingBot), true);
+});

library/vulnerabilities/bot-spoofing/verifyBotAuthenticity.ts

@@ -0,0 +1,24 @@
+import type { ServiceConfigBotSpoofingData } from "../../agent/ServiceConfig";
+import { verifyBotAuthenticityWithDNS } from "./verifyBotAuthenticityWithDNS";
+
+export async function verifyBotAuthenticity(
+  requestIp: string,
+  matchingBot: ServiceConfigBotSpoofingData
+) {
+  // Check if the IP address matches any of the whitelisted IP addresses
+  if (matchingBot.ips) {
+    if (matchingBot.ips.has(requestIp)) {
+      return true;
+    }
+  }
+
+  if (matchingBot.hostnames.length > 0) {
+    // Check if the hostname matches any of the whitelisted hostnames
+    if (await verifyBotAuthenticityWithDNS(requestIp, matchingBot)) {
+      // Todo cache
+      return true;
+    }
+  }
+
+  return false;
+}

library/vulnerabilities/bot-spoofing/verifyBotAuthenticityWithDNS.ts

@@ -0,0 +1,63 @@
+import { resolve, reverse } from "dns/promises";
+import type { ServiceConfigBotSpoofingData } from "../../agent/ServiceConfig";
+import { getInstance } from "../../agent/AgentSingleton";
+
+/**
+ * Checks the authenticity of a bot by performing a reverse DNS lookup on the request IP address.
+ * Returns true if the bot is authentic, false otherwise.
+ *
+ * Example:
+ *
+ * 1. Do a reverse DNS lookup of the request ip
+ *   e.g. `54.236.1.12` → crawl-54-236-1-12.pinterest.com
+ * 2. Check if the domain matches the expected origin
+ *   e.g. Domain is pinterest.com or pinterestcrawler.com for `Pinterestbot`
+ * 3. Because PTR records can be spoofed, also lookup the A and AAAA record and compare them with the request ip:
+ *   e.g. `crawl-54-236-1-12.pinterest.com` → `54.236.1.12`
+ */
+export async function verifyBotAuthenticityWithDNS(
+  requestIp: string,
+  matchingBot: ServiceConfigBotSpoofingData
+) {
+  try {
+    // Send a reverse DNS lookup request
+    const hostnames = await reverse(requestIp);
+    if (!Array.isArray(hostnames)) {
+      // No PTR record found
+      return false;
+    }
+
+    // Filter out hostnames that don't end with any of the whitelisted hostnames
+    const matchingHostnames = hostnames.filter((hostname) =>
+      matchingBot.hostnames.some((whitelistedHostname) =>
+        // Check if the hostname ends with the whitelisted hostname
+        hostname.endsWith(`.${whitelistedHostname}`)
+      )
+    );
+
+    if (matchingHostnames.length === 0) {
+      // No matching hostnames found, so the bot is not authentic
+      return false;
+    }
+
+    const rrType = requestIp.includes(":") ? "AAAA" : "A";
+
+    // Check if the IP address matches any of the A or AAAA records for the matching hostnames
+    for (const hostname of matchingHostnames) {
+      const addresses = await resolve(hostname, rrType);
+      if (!Array.isArray(addresses)) {
+        // No A or AAAA records found
+        continue;
+      }
+      if (addresses.some((address) => address === requestIp)) {
+        // The IP address matches the A or AAAA record for the hostname
+        return true;
+      }
+    }
+
+    return false;
+  } catch (error) {
+    getInstance()?.log(`Bot Spoofing Protection: DNS check failed: ${error}`);
+    return true; // Fallback to true on error to prevent blocking legitimate requests
+  }
+}