Adapt to new firewall/lists API

Author: hansott

library/agent/Agent.test.ts

@@ -33,27 +33,31 @@ wrap(fetch, "fetch", function mock() {
             source: "name",
             description: "Description",
             ips: ["1.3.2.0/24", "fe80::1234:5678:abcd:ef12/64"],
-            monitor: false,
-          },
-        ],
-        blockedUserAgents: [
-          {
-            key: "ai_bots",
-            pattern: "AI2Bot|Bytespider",
-            monitor: false,
           },
         ],
+        blockedUserAgents: "AI2Bot|Bytespider",
         allowedIPAddresses: shouldOnlyAllowSomeIPAddresses
           ? [
               {
                 key: "some/key",
                 source: "name",
                 description: "Description",
                 ips: ["4.3.2.1"],
-                monitor: false,
               },
             ]
           : [],
+        monitoredIPAddresses: [],
+        monitoredUserAgents: "",
+        userAgentDetails: [
+          {
+            key: "AI2Bot",
+            pattern: "AI2Bot",
+          },
+          {
+            key: "Bytespider",
+            pattern: "Bytespider",
+          },
+        ],
       } satisfies Response),
     };
   };
@@ -1076,51 +1080,36 @@ t.test("it fetches blocked lists", async () => {
 
   await setTimeout(0);
 
-  t.same(agent.getConfig().getBlockedIPAddresses("1.3.2.4"), [
-    {
-      key: "some/key",
-      monitor: false,
-      reason: "Description",
-    },
-  ]);
-  t.same(agent.getConfig().getBlockedIPAddresses("fe80::1234:5678:abcd:ef12"), [
-    {
-      key: "some/key",
-      monitor: false,
-      reason: "Description",
-    },
-  ]);
+  t.same(agent.getConfig().isIPAddressBlocked("1.3.2.4"), {
+    blocked: true,
+    reason: "Description",
+  });
+  t.same(agent.getConfig().isIPAddressBlocked("fe80::1234:5678:abcd:ef12"), {
+    blocked: true,
+    reason: "Description",
+  });
 
   t.same(
     agent
       .getConfig()
-      .getBlockedUserAgents(
+      .isUserAgentBlocked(
         "Mozilla/5.0 (compatible) AI2Bot (+https://www.allenai.org/crawler)"
       ),
-    [
-      {
-        key: "ai_bots",
-        monitor: false,
-      },
-    ]
+    {
+      blocked: true,
+    }
   );
 
   t.same(
-    agent
-      .getConfig()
-      .getBlockedUserAgents("Mozilla/5.0 (compatible) Bytespider"),
-    [
-      {
-        key: "ai_bots",
-        monitor: false,
-      },
-    ]
+    agent.getConfig().isUserAgentBlocked("Mozilla/5.0 (compatible) Bytespider"),
+    {
+      blocked: true,
+    }
   );
 
-  t.same(
-    agent.getConfig().getBlockedUserAgents("Mozilla/5.0 (compatible)"),
-    []
-  );
+  t.same(agent.getConfig().isUserAgentBlocked("Mozilla/5.0 (compatible)"), {
+    blocked: false,
+  });
 });
 
 t.test("it does not fetch blocked IPs if serverless", async () => {
@@ -1134,18 +1123,23 @@ t.test("it does not fetch blocked IPs if serverless", async () => {
 
   await setTimeout(0);
 
-  t.same(agent.getConfig().getBlockedIPAddresses("1.3.2.4"), []);
+  t.same(agent.getConfig().isIPAddressBlocked("1.3.2.4"), {
+    blocked: false,
+  });
+
   t.same(agent.getConfig().isAllowedIPAddress("1.3.2.4"), {
     allowed: true,
   });
 
   t.same(
     agent
       .getConfig()
-      .getBlockedUserAgents(
+      .isUserAgentBlocked(
         "Mozilla/5.0 (compatible) AI2Bot (+https://www.allenai.org/crawler)"
       ),
-    []
+    {
+      blocked: false,
+    }
   );
 });
 
@@ -1160,20 +1154,14 @@ t.test("it only allows some IP addresses", async () => {
 
   await setTimeout(0);
 
-  t.same(agent.getConfig().getBlockedIPAddresses("1.3.2.4"), [
-    {
-      key: "some/key",
-      monitor: false,
-      reason: "Description",
-    },
-  ]);
-  t.same(agent.getConfig().getBlockedIPAddresses("fe80::1234:5678:abcd:ef12"), [
-    {
-      key: "some/key",
-      monitor: false,
-      reason: "Description",
-    },
-  ]);
+  t.same(agent.getConfig().isIPAddressBlocked("1.3.2.4"), {
+    blocked: true,
+    reason: "Description",
+  });
+  t.same(agent.getConfig().isIPAddressBlocked("fe80::1234:5678:abcd:ef12"), {
+    blocked: true,
+    reason: "Description",
+  });
 
   t.same(agent.getConfig().isAllowedIPAddress("1.2.3.4"), {
     allowed: false,

library/agent/Agent.ts

@@ -380,11 +380,20 @@ export class Agent {
     }
 
     try {
-      const { blockedIPAddresses, blockedUserAgents, allowedIPAddresses } =
-        await fetchBlockedLists(this.token);
+      const {
+        blockedIPAddresses,
+        blockedUserAgents,
+        allowedIPAddresses,
+        monitoredIPAddresses,
+        monitoredUserAgents,
+        userAgentDetails,
+      } = await fetchBlockedLists(this.token);
       this.serviceConfig.updateBlockedIPAddresses(blockedIPAddresses);
       this.serviceConfig.updateBlockedUserAgents(blockedUserAgents);
       this.serviceConfig.updateAllowedIPAddresses(allowedIPAddresses);
+      this.serviceConfig.updateMonitoredIPAddresses(monitoredIPAddresses);
+      this.serviceConfig.updateMonitoredUserAgents(monitoredUserAgents);
+      this.serviceConfig.updateUserAgentDetails(userAgentDetails);
     } catch (error: any) {
       console.error(`Aikido: Failed to update blocked lists: ${error.message}`);
     }

library/agent/InspectionStatistics.test.ts

@@ -581,58 +581,16 @@ t.test("it keeps track of aborted requests", async () => {
   clock.uninstall();
 });
 
-t.test("it keeps track of blocked IPs and user agents", async () => {
+t.test("it keeps track of matched IPs and user agents", async () => {
   const clock = FakeTimers.install();
 
   const stats = new InspectionStatistics({
     maxPerfSamplesInMemory: 50,
     maxCompressedStatsInMemory: 5,
   });
 
-  stats.onIPAddressMatches([
-    { key: "known_threat_actors/public_scanners", monitor: false },
-  ]);
-  stats.onUserAgentMatches([{ key: "ai_bots", monitor: false }]);
-
-  t.same(stats.getStats(), {
-    operations: {},
-    startedAt: 0,
-    requests: {
-      total: 0,
-      aborted: 0,
-      attacksDetected: {
-        total: 0,
-        blocked: 0,
-      },
-    },
-    userAgents: {
-      breakdown: {
-        // eslint-disable-next-line camelcase
-        ai_bots: { total: 1, blocked: 1 },
-      },
-    },
-    ipAddresses: {
-      breakdown: {
-        "known_threat_actors/public_scanners": { total: 1, blocked: 1 },
-      },
-    },
-  });
-
-  clock.uninstall();
-});
-
-t.test("it keeps track of monitored IPs and user agents", async () => {
-  const clock = FakeTimers.install();
-
-  const stats = new InspectionStatistics({
-    maxPerfSamplesInMemory: 50,
-    maxCompressedStatsInMemory: 5,
-  });
-
-  stats.onIPAddressMatches([
-    { key: "known_threat_actors/public_scanners", monitor: true },
-  ]);
-  stats.onUserAgentMatches([{ key: "ai_data_scrapers", monitor: true }]);
+  stats.onIPAddressMatches(["known_threat_actors/public_scanners"]);
+  stats.onUserAgentMatches(["ai_data_scrapers"]);
 
   t.same(stats.getStats(), {
     operations: {},
@@ -659,10 +617,8 @@ t.test("it keeps track of monitored IPs and user agents", async () => {
   });
 
   // Test multiple occurrences
-  stats.onIPAddressMatches([
-    { key: "known_threat_actors/public_scanners", monitor: true },
-  ]);
-  stats.onUserAgentMatches([{ key: "ai_data_scrapers", monitor: true }]);
+  stats.onIPAddressMatches(["known_threat_actors/public_scanners"]);
+  stats.onUserAgentMatches(["ai_data_scrapers"]);
 
   t.same(stats.getStats(), {
     operations: {},
@@ -691,42 +647,6 @@ t.test("it keeps track of monitored IPs and user agents", async () => {
   clock.uninstall();
 });
 
-t.test("should track multiple matches for the same key", (t) => {
-  const clock = FakeTimers.install();
-
-  const stats = new InspectionStatistics({
-    maxPerfSamplesInMemory: 100,
-    maxCompressedStatsInMemory: 10,
-  });
-
-  stats.onIPAddressMatches([
-    { key: "known_threat_actors/public_scanners", monitor: true },
-    { key: "known_threat_actors/public_scanners", monitor: false },
-  ]);
-  stats.onUserAgentMatches([
-    { key: "ai_data_scrapers", monitor: true },
-    { key: "ai_data_scrapers", monitor: false },
-  ]);
-
-  const result = stats.getStats();
-
-  t.equal(
-    result.ipAddresses.breakdown["known_threat_actors/public_scanners"].total,
-    2
-  );
-  t.equal(
-    result.ipAddresses.breakdown["known_threat_actors/public_scanners"].blocked,
-    1
-  );
-
-  t.equal(result.userAgents.breakdown["ai_data_scrapers"].total, 2);
-  t.equal(result.userAgents.breakdown["ai_data_scrapers"].blocked, 1);
-
-  t.end();
-
-  clock.uninstall();
-});
-
 t.test("it keeps track of multiple operations of the same kind", async () => {
   const clock = FakeTimers.install();
 

library/agent/InspectionStatistics.ts

@@ -27,11 +27,11 @@ type UserAgentBotKey = string;
 type IPListKey = string;
 
 type UserAgentStats = {
-  breakdown: Record<UserAgentBotKey, { total: number; blocked: number }>;
+  breakdown: Record<UserAgentBotKey, number>;
 };
 
 type IPAddressStats = {
-  breakdown: Record<IPListKey, { total: number; blocked: number }>;
+  breakdown: Record<IPListKey, number>;
 };
 
 export class InspectionStatistics {
@@ -111,10 +111,10 @@ export class InspectionStatistics {
       };
     };
     userAgents: {
-      breakdown: Record<string, { total: number; blocked: number }>;
+      breakdown: Record<string, number>;
     };
     ipAddresses: {
-      breakdown: Record<string, { total: number; blocked: number }>;
+      breakdown: Record<string, number>;
     };
   } {
     const operations: Record<string, OperationStatsWithoutTimings> = {};
@@ -222,31 +222,23 @@ export class InspectionStatistics {
     }
   }
 
-  onIPAddressMatches(matches: { key: IPListKey; monitor: boolean }[]) {
-    matches.forEach((match) => {
-      if (!this.ipAddresses.breakdown[match.key]) {
-        this.ipAddresses.breakdown[match.key] = { total: 0, blocked: 0 };
+  onIPAddressMatches(matches: IPListKey[]) {
+    matches.forEach((key) => {
+      if (!this.ipAddresses.breakdown[key]) {
+        this.ipAddresses.breakdown[key] = 0;
       }
 
-      this.ipAddresses.breakdown[match.key].total += 1;
-
-      if (!match.monitor) {
-        this.ipAddresses.breakdown[match.key].blocked += 1;
-      }
+      this.ipAddresses.breakdown[key] += 1;
     });
   }
 
-  onUserAgentMatches(matches: { key: UserAgentBotKey; monitor: boolean }[]) {
-    matches.forEach((match) => {
-      if (!this.userAgents.breakdown[match.key]) {
-        this.userAgents.breakdown[match.key] = { total: 0, blocked: 0 };
+  onUserAgentMatches(matches: UserAgentBotKey[]) {
+    matches.forEach((key) => {
+      if (!this.userAgents.breakdown[key]) {
+        this.userAgents.breakdown[key] = 0;
       }
 
-      this.userAgents.breakdown[match.key].total += 1;
-
-      if (!match.monitor) {
-        this.userAgents.breakdown[match.key].blocked += 1;
-      }
+      this.userAgents.breakdown[key] += 1;
     });
   }