Report stats about blocked/monitored user agents and IPs

end2end/server/src/handlers/lists.js

@@ -20,22 +20,37 @@ module.exports = function lists(req, res) {
       blockedIps.length > 0
         ? [
             {
+              key: "geoip/Belgium;BE",
               source: "geoip",
               description: "geo restrictions",
               ips: blockedIps,
+              monitor: false,
+            },
+          ]
+        : [],
+    blockedUserAgents:
+      blockedUserAgents.length > 0
+        ? [
+            {
+              key: "hackers",
+              pattern: blockedUserAgents,
+              monitor: false,
             },
           ]
         : [],
-    blockedUserAgents: blockedUserAgents,
     allowedIPAddresses:
       allowedIps.length > 0
         ? [
             {
+              key: "geoip/Belgium;BE",
               source: "geoip",
               description: "geo restrictions",
               ips: allowedIps,
+              monitor: false,
             },
           ]
         : [],
+    monitoredIPAddresses: [],
+    monitoredUserAgents: [],
   });
 };

end2end/server/src/zen/config.js

@@ -90,7 +90,7 @@ function getAllowedIPAddresses(app) {
 }
 
 function updateBlockedUserAgents(app, uas) {
-  let entry = blockedUserAgents.find((e) => e.serviceId === e.serviceId);
+  let entry = blockedUserAgents.find((e) => e.serviceId === app.serviceId);
 
   if (entry) {
     entry.userAgents = uas;
@@ -104,13 +104,13 @@ function updateBlockedUserAgents(app, uas) {
 }
 
 function getBlockedUserAgents(app) {
-  const entry = blockedUserAgents.find((e) => e.serviceId === e.serviceId);
+  const entry = blockedUserAgents.find((e) => e.serviceId === app.serviceId);
 
   if (entry) {
     return entry.userAgents;
   }
 
-  return "";
+  return { serviceId: app.serviceId, userAgents: [] };
 }
 
 module.exports = {

library/agent/Agent.test.ts

@@ -18,6 +18,7 @@ import { Wrapper } from "./Wrapper";
 import { Context } from "./Context";
 import { createTestAgent } from "../helpers/createTestAgent";
 import { setTimeout } from "node:timers/promises";
+import type { Response } from "./api/fetchBlockedLists";
 
 let shouldOnlyAllowSomeIPAddresses = false;
 
@@ -28,22 +29,32 @@ wrap(fetch, "fetch", function mock() {
       body: JSON.stringify({
         blockedIPAddresses: [
           {
+            key: "some/key",
             source: "name",
             description: "Description",
             ips: ["1.3.2.0/24", "fe80::1234:5678:abcd:ef12/64"],
+            monitor: false,
+          },
+        ],
+        blockedUserAgents: [
+          {
+            key: "ai_bots",
+            pattern: "AI2Bot|Bytespider",
+            monitor: false,
           },
         ],
-        blockedUserAgents: "AI2Bot|Bytespider",
         allowedIPAddresses: shouldOnlyAllowSomeIPAddresses
           ? [
               {
+                key: "some/key",
                 source: "name",
                 description: "Description",
                 ips: ["4.3.2.1"],
+                monitor: false,
               },
             ]
           : [],
-      }),
+      } satisfies Response),
     };
   };
 });
@@ -1065,36 +1076,51 @@ t.test("it fetches blocked lists", async () => {
 
   await setTimeout(0);
 
-  t.same(agent.getConfig().isIPAddressBlocked("1.3.2.4"), {
-    blocked: true,
-    reason: "Description",
-  });
-  t.same(agent.getConfig().isIPAddressBlocked("fe80::1234:5678:abcd:ef12"), {
-    blocked: true,
-    reason: "Description",
-  });
+  t.same(agent.getConfig().getBlockedIPAddresses("1.3.2.4"), [
+    {
+      key: "some/key",
+      monitor: false,
+      reason: "Description",
+    },
+  ]);
+  t.same(agent.getConfig().getBlockedIPAddresses("fe80::1234:5678:abcd:ef12"), [
+    {
+      key: "some/key",
+      monitor: false,
+      reason: "Description",
+    },
+  ]);
 
   t.same(
     agent
       .getConfig()
-      .isUserAgentBlocked(
+      .getBlockedUserAgents(
         "Mozilla/5.0 (compatible) AI2Bot (+https://www.allenai.org/crawler)"
       ),
-    {
-      blocked: true,
-    }
+    [
+      {
+        key: "ai_bots",
+        monitor: false,
+      },
+    ]
   );
 
   t.same(
-    agent.getConfig().isUserAgentBlocked("Mozilla/5.0 (compatible) Bytespider"),
-    {
-      blocked: true,
-    }
+    agent
+      .getConfig()
+      .getBlockedUserAgents("Mozilla/5.0 (compatible) Bytespider"),
+    [
+      {
+        key: "ai_bots",
+        monitor: false,
+      },
+    ]
   );
 
-  t.same(agent.getConfig().isUserAgentBlocked("Mozilla/5.0 (compatible)"), {
-    blocked: false,
-  });
+  t.same(
+    agent.getConfig().getBlockedUserAgents("Mozilla/5.0 (compatible)"),
+    []
+  );
 });
 
 t.test("it does not fetch blocked IPs if serverless", async () => {
@@ -1108,23 +1134,18 @@ t.test("it does not fetch blocked IPs if serverless", async () => {
 
   await setTimeout(0);
 
-  t.same(agent.getConfig().isIPAddressBlocked("1.3.2.4"), {
-    blocked: false,
-  });
-
+  t.same(agent.getConfig().getBlockedIPAddresses("1.3.2.4"), []);
   t.same(agent.getConfig().isAllowedIPAddress("1.3.2.4"), {
     allowed: true,
   });
 
   t.same(
     agent
       .getConfig()
-      .isUserAgentBlocked(
+      .getBlockedUserAgents(
         "Mozilla/5.0 (compatible) AI2Bot (+https://www.allenai.org/crawler)"
       ),
-    {
-      blocked: false,
-    }
+    []
   );
 });
 
@@ -1139,14 +1160,20 @@ t.test("it only allows some IP addresses", async () => {
 
   await setTimeout(0);
 
-  t.same(agent.getConfig().isIPAddressBlocked("1.3.2.4"), {
-    blocked: true,
-    reason: "Description",
-  });
-  t.same(agent.getConfig().isIPAddressBlocked("fe80::1234:5678:abcd:ef12"), {
-    blocked: true,
-    reason: "Description",
-  });
+  t.same(agent.getConfig().getBlockedIPAddresses("1.3.2.4"), [
+    {
+      key: "some/key",
+      monitor: false,
+      reason: "Description",
+    },
+  ]);
+  t.same(agent.getConfig().getBlockedIPAddresses("fe80::1234:5678:abcd:ef12"), [
+    {
+      key: "some/key",
+      monitor: false,
+      reason: "Description",
+    },
+  ]);
 
   t.same(agent.getConfig().isAllowedIPAddress("1.2.3.4"), {
     allowed: false,

library/agent/Agent.ts

@@ -312,6 +312,8 @@ export class Agent {
             startedAt: stats.startedAt,
             endedAt: endedAt,
             requests: stats.requests,
+            userAgents: stats.userAgents,
+            ipAddresses: stats.ipAddresses,
           },
           hostnames: outgoingDomains,
           routes: routes,