Merge branch 'main' into bot-spoofing-protection

Author: timokoessler

.github/workflows/end-to-end-tests.yml

@@ -45,7 +45,7 @@ jobs:
           "CLICKHOUSE_DEFAULT_ACCESS": "MANAGEMENT=1"
         ports:
           - "27019:8123"
-    timeout-minutes: 10
+    timeout-minutes: 15
     strategy:
       matrix:
         node-version: [18.x]

.github/workflows/unit-test.yml

@@ -52,7 +52,7 @@ jobs:
         ports:
           - "27019:8123"
       mongodb-replica:
-        image: bitnami/mongodb:8.0
+        image: bitnami/mongodb:4.4
         env:
           MONGODB_ADVERTISED_HOSTNAME: 127.0.0.1
           MONGODB_REPLICA_SET_MODE: primary
@@ -77,6 +77,14 @@ jobs:
       - name: Add local.aikido.io to /etc/hosts
         run: |
           sudo echo "127.0.0.1 local.aikido.io" | sudo tee -a /etc/hosts
+      - name: "Install Google Cloud SDK"
+        uses: google-github-actions/setup-gcloud@6189d56e4096ee891640bb02ac264be376592d6a # v2
+        with:
+          install_components: "beta,pubsub-emulator"
+          version: "524.0.0"
+      - name: "Start Pub/Sub emulator"
+        run: |
+          gcloud beta emulators pubsub start --project=sample-project --host-port='0.0.0.0:8085' &
       - run: npm run install-lib-only
       - run: npm run build
       - run: npm run test:ci

README.md

@@ -66,7 +66,7 @@ Zen for Node.js 16+ is compatible with:
 ### Cloud providers
 
 * ✅ [`@google-cloud/functions-framework`](https://www.npmjs.com/package/@google-cloud/functions-framework) 4.x, 3.x
-* ✅ [`@google-cloud/pubsub`](https://www.npmjs.com/package/@google-cloud/pubsub) 4.x
+* ✅ [`@google-cloud/pubsub`](https://www.npmjs.com/package/@google-cloud/pubsub) 5.x, 4.x
 * ✅ Google Cloud Functions
 * ✅ AWS Lambda
 

end2end/server/app.js

@@ -8,6 +8,7 @@ const checkToken = require("./src/middleware/checkToken");
 const updateConfig = require("./src/handlers/updateConfig");
 const lists = require("./src/handlers/lists");
 const updateLists = require("./src/handlers/updateLists");
+const realtimeConfig = require("./src/handlers/realtimeConfig");
 
 const app = express();
 
@@ -18,6 +19,9 @@ app.use(express.json());
 app.get("/api/runtime/config", checkToken, config);
 app.post("/api/runtime/config", checkToken, updateConfig);
 
+// Realtime polling endpoint
+app.get("/config", checkToken, realtimeConfig);
+
 app.get("/api/runtime/events", checkToken, listEvents);
 app.post("/api/runtime/events", checkToken, captureEvent);
 

end2end/server/src/handlers/realtimeConfig.js

@@ -0,0 +1,14 @@
+const { getAppConfig } = require("../zen/config");
+
+module.exports = function realtimeConfig(req, res) {
+  if (!req.app) {
+    throw new Error("App is missing");
+  }
+
+  const config = getAppConfig(req.app);
+
+  res.json({
+    serviceId: req.app.serviceId,
+    configUpdatedAt: config.configUpdatedAt,
+  });
+};

end2end/server/src/zen/config.js

@@ -9,7 +9,7 @@ function generateConfig(app) {
     endpoints: [],
     blockedUserIds: [],
     allowedIPAddresses: [],
-    receivedAnyStats: true,
+    receivedAnyStats: false,
   };
 }
 

end2end/server/src/zen/events.js

@@ -1,11 +1,6 @@
 const events = new Map();
 
 function captureEvent(event, app) {
-  if (event.type === "heartbeat") {
-    // Ignore heartbeats
-    return;
-  }
-
   if (!events.has(app.id)) {
     events.set(app.id, []);
   }

end2end/tests/express-mongodb.ssrf.test.js

@@ -58,6 +58,7 @@ t.test("it blocks in blocking mode", (t) => {
       AIKIDO_BLOCKING: "true",
       AIKIDO_TOKEN: token,
       AIKIDO_ENDPOINT: testServerUrl,
+      AIKIDO_REALTIME_ENDPOINT: testServerUrl,
     },
   });
 
@@ -156,6 +157,7 @@ t.test("it does not block in dry mode", (t) => {
       AIKIDO_DEBUG: "true",
       AIKIDO_TOKEN: token,
       AIKIDO_ENDPOINT: testServerUrl,
+      AIKIDO_REALTIME_ENDPOINT: testServerUrl,
     },
   });
 
@@ -214,6 +216,7 @@ t.test("it blocks request to base URL if proxy is not trusted", (t) => {
       AIKIDO_BLOCKING: "true",
       AIKIDO_TOKEN: token,
       AIKIDO_ENDPOINT: testServerUrl,
+      AIKIDO_REALTIME_ENDPOINT: testServerUrl,
       AIKIDO_TRUST_PROXY: "false",
     },
   });

end2end/tests/express-mysql.test.js

@@ -9,6 +9,17 @@ const pathToApp = resolve(
   "app.js"
 );
 
+const testServerUrl = "http://localhost:5874";
+
+let token;
+t.beforeEach(async () => {
+  const response = await fetch(`${testServerUrl}/api/runtime/apps`, {
+    method: "POST",
+  });
+  const body = await response.json();
+  token = body.token;
+});
+
 t.test("it blocks in blocking mode", (t) => {
   const server = spawn(`node`, [pathToApp, "4000"], {
     env: { ...process.env, AIKIDO_DEBUG: "true", AIKIDO_BLOCKING: "true" },
@@ -144,3 +155,86 @@ t.test("it does not block in dry mode", (t) => {
       server.kill();
     });
 });
+
+t.setTimeout(80000);
+
+t.test(
+  "it increments sqlTokenizationFailures counter for invalid SQL queries",
+  (t) => {
+    const server = spawn(`node`, [pathToApp, "4003"], {
+      env: {
+        ...process.env,
+        AIKIDO_DEBUG: "true",
+        AIKIDO_BLOCKING: "true",
+        AIKIDO_TOKEN: token,
+        AIKIDO_ENDPOINT: testServerUrl,
+        AIKIDO_REALTIME_ENDPOINT: testServerUrl,
+      },
+    });
+
+    server.on("close", () => {
+      t.end();
+    });
+
+    server.on("error", (err) => {
+      t.fail(err);
+    });
+
+    let stdout = "";
+    server.stdout.on("data", (data) => {
+      stdout += data.toString();
+    });
+
+    let stderr = "";
+    server.stderr.on("data", (data) => {
+      stderr += data.toString();
+    });
+
+    // Wait for the server to start
+    timeout(2000)
+      .then(() => {
+        return fetch(
+          `http://localhost:4003/invalid-query?sql=${encodeURIComponent("SELECT * FROM test")}`,
+          {
+            signal: AbortSignal.timeout(5000),
+            method: "POST",
+          }
+        );
+      })
+      .then((response) => {
+        return response.text();
+      })
+      .then((responseText) => {
+        t.match(responseText, /You have an error in your SQL syntax/);
+
+        // Wait for the heartbeat event to be sent
+        return timeout(60000);
+      })
+      .then(() => {
+        return fetch(`${testServerUrl}/api/runtime/events`, {
+          method: "GET",
+          headers: {
+            Authorization: token,
+          },
+          signal: AbortSignal.timeout(5000),
+        });
+      })
+      .then((response) => {
+        return response.json();
+      })
+      .then((events) => {
+        const heartbeatEvents = events.filter(
+          (event) => event.type === "heartbeat"
+        );
+        t.same(heartbeatEvents.length, 1);
+        const [heartbeat] = heartbeatEvents;
+        t.equal(heartbeat.stats.sqlTokenizationFailures, 1);
+      })
+      .catch((error) => {
+        t.error(error);
+      })
+      .finally(() => {
+        server.kill();
+      });
+  }
+);

end2end/tests/hono-xml-allowlists.test.ts

@@ -78,6 +78,7 @@ t.test("it blocks non-allowed IP addresses", (t) => {
       AIKIDO_BLOCK: "true",
       AIKIDO_TOKEN: token,
       AIKIDO_ENDPOINT: testServerUrl,
+      AIKIDO_REALTIME_ENDPOINT: testServerUrl,
     },
   });