Merge pull request #454 from AikidoSec/poc-required-pkg

Report packages used at runtime

Author: hansott

library/agent/Agent.ts

@@ -25,6 +25,7 @@ import { wrapInstalledPackages } from "./wrapInstalledPackages";
 import { Wrapper } from "./Wrapper";
 import { isAikidoCI } from "../helpers/isAikidoCI";
 import { AttackLogger } from "./AttackLogger";
+import { Packages } from "./Packages";
 
 type WrappedPackage = { version: string | null; supported: boolean };
 
@@ -38,6 +39,7 @@ export class Agent {
   private preventedPrototypePollution = false;
   private incompatiblePackages: Record<string, string> = {};
   private wrappedPackages: Record<string, WrappedPackage> = {};
+  private packages = new Packages();
   private timeoutInMS = 30 * 1000;
   private hostnames = new Hostnames(200);
   private users = new Users(1000);
@@ -296,11 +298,13 @@ export class Agent {
       const routes = this.routes.asArray();
       const outgoingDomains = this.hostnames.asArray();
       const users = this.users.asArray();
+      const packages = this.packages.asArray();
       const endedAt = Date.now();
       this.statistics.reset();
       this.routes.clear();
       this.hostnames.clear();
       this.users.clear();
+      this.packages.clear();
       const response = await this.api.report(
         this.token,
         {
@@ -315,6 +319,7 @@ export class Agent {
             userAgents: stats.userAgents,
             ipAddresses: stats.ipAddresses,
           },
+          packages,
           hostnames: outgoingDomains,
           routes: routes,
           users: users,
@@ -479,6 +484,10 @@ export class Agent {
       }
     }
 
+    // When our library is required, we are not intercepting `require` calls yet
+    // We need to add our library to the list of packages manually
+    this.onPackageRequired("@aikido/firewall", getAgentVersion());
+
     wrapInstalledPackages(wrappers, this.serverless);
 
     // Send startup event and wait for config
@@ -503,11 +512,19 @@ export class Agent {
     this.logger.log(`Failed to wrap module ${module}: ${error.message}`);
   }
 
+  onPackageRequired(name: string, version: string) {
+    this.packages.addPackage({
+      name,
+      version,
+    });
+  }
+
   onPackageWrapped(name: string, details: WrappedPackage) {
     if (this.wrappedPackages[name]) {
       // Already reported as wrapped
       return;
     }
+
     this.wrappedPackages[name] = details;
 
     if (details.version) {

library/agent/Packages.test.ts

@@ -0,0 +1,76 @@
+import * as t from "tap";
+import * as FakeTimers from "@sinonjs/fake-timers";
+import { Packages } from "./Packages";
+
+t.test("addPackage should add a new package", async (t) => {
+  const clock = FakeTimers.install();
+  const packages = new Packages();
+  packages.addPackage({ name: "express", version: "4.17.1" });
+  const arr = packages.asArray();
+  t.same(arr, [{ name: "express", version: "4.17.1", requiredAt: 0 }]);
+  clock.uninstall();
+});
+
+t.test(
+  "addPackage should add a new version for an existing package",
+  async (t) => {
+    const clock = FakeTimers.install();
+    const packages = new Packages();
+    packages.addPackage({ name: "lodash", version: "4.17.20" });
+    clock.tick(10);
+    packages.addPackage({ name: "lodash", version: "4.17.21" });
+    const arr = packages.asArray();
+    t.same(arr, [
+      { name: "lodash", version: "4.17.20", requiredAt: 0 },
+      { name: "lodash", version: "4.17.21", requiredAt: 10 },
+    ]);
+    clock.uninstall();
+  }
+);
+
+t.test("addPackage should not add a duplicate package version", async (t) => {
+  const clock = FakeTimers.install();
+  const packages = new Packages();
+  packages.addPackage({ name: "moment", version: "2.29.1" });
+  packages.addPackage({ name: "moment", version: "2.29.1" });
+  const arr = packages.asArray();
+  t.same(arr, [{ name: "moment", version: "2.29.1", requiredAt: 0 }]);
+  clock.uninstall();
+});
+
+t.test(
+  "asArray should return an empty array when no packages are added",
+  async (t) => {
+    const packages = new Packages();
+    const arr = packages.asArray();
+    t.same(arr, [], "should return an empty array");
+  }
+);
+
+t.test("asArray should return all packages and versions", async (t) => {
+  const clock = FakeTimers.install();
+  const packages = new Packages();
+  packages.addPackage({ name: "express", version: "4.17.1" });
+  clock.tick(5);
+  packages.addPackage({ name: "lodash", version: "4.17.20" });
+  clock.tick(15);
+  packages.addPackage({ name: "lodash", version: "4.17.21" });
+  const arr = packages.asArray();
+  t.same(arr, [
+    { name: "express", version: "4.17.1", requiredAt: 0 },
+    { name: "lodash", version: "4.17.20", requiredAt: 5 },
+    { name: "lodash", version: "4.17.21", requiredAt: 20 },
+  ]);
+  clock.uninstall();
+});
+
+t.test("clear should remove all packages", async (t) => {
+  const clock = FakeTimers.install();
+  const packages = new Packages();
+  packages.addPackage({ name: "express", version: "4.17.1" });
+  packages.addPackage({ name: "lodash", version: "4.17.20" });
+  packages.clear();
+  const arr = packages.asArray();
+  t.same(arr, [], "should return an empty array after clear");
+  clock.uninstall();
+});

library/agent/Packages.ts

@@ -0,0 +1,34 @@
+type PackageInfo = {
+  name: string;
+  version: string;
+  requiredAt: number;
+};
+
+export class Packages {
+  private packages: Map<string, PackageInfo[]> = new Map();
+
+  addPackage(pkg: { name: string; version: string }) {
+    const versions = this.packages.get(pkg.name) || [];
+    const existingVersion = versions.find((v) => v.version === pkg.version);
+
+    if (existingVersion) {
+      return;
+    }
+
+    versions.push({
+      name: pkg.name,
+      version: pkg.version,
+      requiredAt: Date.now(),
+    });
+
+    this.packages.set(pkg.name, versions);
+  }
+
+  asArray() {
+    return Array.from(this.packages.values()).flat();
+  }
+
+  clear() {
+    this.packages.clear();
+  }
+}

library/agent/api/Event.ts

@@ -112,6 +112,11 @@ type Heartbeat = {
       breakdown: Record<string, number>;
     };
   };
+  packages: {
+    name: string;
+    version: string;
+    requiredAt: number;
+  }[];
   hostnames: { hostname: string; port: number | undefined; hits: number }[];
   routes: {
     path: string;

library/agent/api/ReportingAPIRateLimitedClientSide.test.ts

@@ -188,6 +188,7 @@ function generateHeartbeatEvent(): Event {
     hostnames: [],
     routes: [],
     users: [],
+    packages: [],
   };
 }
 

library/agent/hooks/wrapRequire.ts

@@ -183,21 +183,21 @@ function patchPackage(this: mod, id: string, originalExports: unknown) {
   const moduleName = pathInfo.name;
 
   // Get all versioned packages for the module name
-  const versionedPackages = packages
+  const versionedPackagesToInstrument = packages
     .filter((pkg) => pkg.getName() === moduleName)
     .map((pkg) => pkg.getVersions())
     .flat();
 
-  // We don't want to patch this package because we do not have any hooks for it
-  if (!versionedPackages.length) {
+  // Read the package.json of the required package
+  let packageJson: PackageJson | undefined;
+  try {
+    packageJson = originalRequire(
+      `${pathInfo.base}/package.json`
+    ) as PackageJson;
+  } catch {
     return originalExports;
   }
 
-  // Read the package.json of the required package
-  const packageJson = originalRequire(
-    `${pathInfo.base}/package.json`
-  ) as PackageJson;
-
   // Get the version of the installed package
   const installedPkgVersion = packageJson.version;
   if (!installedPkgVersion) {
@@ -206,19 +206,24 @@ function patchPackage(this: mod, id: string, originalExports: unknown) {
     );
   }
 
+  const agent = getInstance();
+  agent?.onPackageRequired(moduleName, installedPkgVersion);
+
+  // We don't want to patch this package because we do not have any hooks for it
+  if (!versionedPackagesToInstrument.length) {
+    return originalExports;
+  }
+
   // Check if the installed package version is supported (get all matching versioned packages)
-  const matchingVersionedPackages = versionedPackages.filter((pkg) =>
-    satisfiesVersion(pkg.getRange(), installedPkgVersion)
+  const matchingVersionedPackages = versionedPackagesToInstrument.filter(
+    (pkg) => satisfiesVersion(pkg.getRange(), installedPkgVersion)
   );
 
-  const agent = getInstance();
-  if (agent) {
-    // Report to the agent that the package was wrapped or not if it's version is not supported
-    agent.onPackageWrapped(moduleName, {
-      version: installedPkgVersion,
-      supported: !!matchingVersionedPackages.length,
-    });
-  }
+  // Report to the agent that the package was wrapped or not if it's version is not supported
+  agent?.onPackageWrapped(moduleName, {
+    version: installedPkgVersion,
+    supported: !!matchingVersionedPackages.length,
+  });
 
   if (!matchingVersionedPackages.length) {
     // We don't want to patch this package version

library/sources/Lambda.test.ts

@@ -272,6 +272,7 @@ t.test("it sends heartbeat after first and every 10 minutes", async () => {
       hostnames: [],
       routes: [],
       users: [],
+      packages: [],
       stats: {
         operations: {
           "mongodb.query": {