fix(shared-metrics): reduce memory consumption with many dependencies (#2088)

refs https://jsw.ibm.com/browse/INSTA-59265

Author: kirrg001

packages/shared-metrics/src/dependencies.js

@@ -27,6 +27,8 @@ exports.MAX_DEPENDENCIES = 750;
 /** @type {string} */
 exports.payloadPrefix = 'dependencies';
 
+const MAX_DEPTH_NODE_MODULES = 2;
+
 /** @type {Object.<string, string>} */
 const preliminaryPayload = {};
 
@@ -87,7 +89,13 @@ exports.activate = function activate() {
  * @param {string} packageJsonPath
  */
 function addAllDependencies(dependencyDir, started, packageJsonPath) {
-  addDependenciesFromDir(dependencyDir, () => {
+  addDependenciesFromDir(dependencyDir, 0, () => {
+    // TODO: This check happens AFTER we have already collected the dependencies.
+    //       This is quiet useless for a large dependency tree, because we consume resources to collect
+    //       all the dependencies (fs.stats, fs.readFile etc), but then discard most of them here.
+    //       This is only critical for a very large number of defined dependencies in package.json (vertical).
+    // NOTE: There is an extra protection in the `addDependenciesFromDir` fn to
+    //       limit the depth of traversing node_modules.
     if (Object.keys(preliminaryPayload).length <= exports.MAX_DEPENDENCIES) {
       // @ts-ignore: Cannot redeclare exported variable 'currentPayload'
       exports.currentPayload = preliminaryPayload;
@@ -114,7 +122,11 @@ function addAllDependencies(dependencyDir, started, packageJsonPath) {
  * @param {string} dependencyDir
  * @param {() => void} callback
  */
-function addDependenciesFromDir(dependencyDir, callback) {
+function addDependenciesFromDir(dependencyDir, currentDepth = 0, callback) {
+  if (currentDepth >= MAX_DEPTH_NODE_MODULES) {
+    return callback();
+  }
+
   fs.readdir(dependencyDir, (readDirErr, dependencies) => {
     if (readDirErr || !dependencies) {
       logger.warn(`Cannot analyse dependencies due to ${readDirErr?.message}`);
@@ -140,11 +152,13 @@ function addDependenciesFromDir(dependencyDir, callback) {
 
     filteredDependendencies.forEach(dependency => {
       if (dependency.indexOf('@') === 0) {
-        addDependenciesFromDir(path.join(dependencyDir, dependency), () => {
+        // NOTE: We do not increase currentDepth because scoped packages are just a folder containing more packages.
+        addDependenciesFromDir(path.join(dependencyDir, dependency), currentDepth, () => {
           countDownLatch.countDown();
         });
       } else {
         const fullDirPath = path.join(dependencyDir, dependency);
+
         // Only check directories. For example, yarn adds a .yarn-integrity file to /node_modules/ which we need to
         // exclude, otherwise we get a confusing "Failed to identify version of .yarn-integrity dependency due to:
         // ENOTDIR: not a directory, open '.../node_modules/.yarn-integrity/package.json'." in the logs.
@@ -159,7 +173,7 @@ function addDependenciesFromDir(dependencyDir, callback) {
             return;
           }
 
-          addDependency(dependency, fullDirPath, countDownLatch);
+          addDependency(dependency, fullDirPath, countDownLatch, currentDepth);
         });
       }
     });
@@ -173,9 +187,11 @@ function addDependenciesFromDir(dependencyDir, callback) {
  * @param {string} dependency
  * @param {string} dependencyDirPath
  * @param {import('./util/CountDownLatch')} countDownLatch
+ * @param {number} currentDepth
  */
-function addDependency(dependency, dependencyDirPath, countDownLatch) {
+function addDependency(dependency, dependencyDirPath, countDownLatch, currentDepth) {
   const packageJsonPath = path.join(dependencyDirPath, 'package.json');
+
   fs.readFile(packageJsonPath, { encoding: 'utf8' }, (err, contents) => {
     if (err && err.code === 'ENOENT') {
       // This directory does not contain a package json. This happens for example for node_modules/.cache etc.
@@ -198,19 +214,23 @@ function addDependency(dependency, dependencyDirPath, countDownLatch) {
         preliminaryPayload[parsedPackageJson.name] = parsedPackageJson.version;
       }
     } catch (parseErr) {
+      // TODO: countDownLatch.countDown(); needs to be called here too?
+      // countDownLatch.countDown();
       return logger.info(
         `Failed to identify version of ${dependency} dependency due to: ${parseErr?.message}.
           This means that you will not be able to see details about this dependency within Instana.`
       );
     }
 
+    // NOTE: The dependency metric collector does not respect if the node_modules are dev dependencies or production
+    //       dependencies. It collects all dependencies that are installed in the node_modules folder.
     const potentialNestedNodeModulesFolder = path.join(dependencyDirPath, 'node_modules');
     fs.stat(potentialNestedNodeModulesFolder, (statErr, stats) => {
       if (statErr || !stats.isDirectory()) {
         countDownLatch.countDown();
         return;
       }
-      addDependenciesFromDir(potentialNestedNodeModulesFolder, () => {
+      addDependenciesFromDir(potentialNestedNodeModulesFolder, currentDepth + 1, () => {
         countDownLatch.countDown();
       });
     });

packages/shared-metrics/test/dependencies/app-with-many-dependencies/app.js

@@ -0,0 +1,33 @@
+/*
+ * (c) Copyright IBM Corp. 2025
+ */
+
+'use strict';
+
+// NOTE: c8 bug https://github.com/bcoe/c8/issues/166
+process.on('SIGTERM', () => {
+  process.disconnect();
+  process.exit(0);
+});
+
+const instana = require('@instana/collector');
+instana();
+
+const express = require('express');
+
+const logPrefix = `Many dependencies app (${process.pid}):\t`;
+
+const app = express();
+
+app.get('/', (req, res) => res.sendStatus(200));
+
+app.listen(process.env.APP_PORT, () => {
+  log(`Listening on port: ${process.env.APP_PORT}`);
+});
+
+function log() {
+  const args = Array.prototype.slice.call(arguments);
+  args[0] = logPrefix + args[0];
+  // eslint-disable-next-line no-console
+  console.log.apply(console, args);
+}

packages/shared-metrics/test/dependencies/app-with-many-dependencies/package.json

@@ -0,0 +1,58 @@
+{
+  "name": "app-with-many-dependencies",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "type": "commonjs",
+  "dependencies": {
+    "@angular/animations": "20.0.3",
+    "@angular/cdk": "20.0.3",
+    "@angular/common": "20.0.3",
+    "@angular/compiler": "20.0.3",
+    "@angular/core": "20.0.3",
+    "@angular/forms": "20.0.3",
+    "@angular/material": "20.0.3",
+    "@angular/material-moment-adapter": "20.0.3",
+    "@angular/platform-browser": "20.0.3",
+    "@angular/platform-browser-dynamic": "20.0.3",
+    "@angular/router": "20.0.3",
+    "@ng-matero/extensions": "20.0.3",
+    "angular13-organization-chart": "2.0.0",
+    "bcm-in-frontend-v2": "file:",
+    "bcm-ph-frontend": "file:",
+    "chart.js": "4.4.1",
+    "chartjs-plugin-zoom": "2.0.1",
+    "compression": "1.8.1",
+    "dayjs": "1.11.7",
+    "dotenv": "16.4.5",
+    "express": "4.21.2",
+    "file-saver": "2.0.5",
+    "flatpickr": "4.6.13",
+    "http-proxy": "1.18.1",
+    "ibmcloud-appid-js": "1.0.1",
+    "keen-slider": "6.8.6",
+    "leaflet": "1.9.3",
+    "lodash-es": "4.17.21",
+    "moment": "2.30.1",
+    "path-to-regexp": "0.1.12",
+    "rxjs": "7.8.1",
+    "tslib": "2.3.0",
+    "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz",
+    "zone.js": "0.15.1"
+  },
+  "devDependencies": {
+    "@angular-devkit/build-angular": "20.0.2",
+    "@angular/cli": "19.2.15",
+    "@angular/compiler-cli": "20.0.3",
+    "@schematics/angular": "18.2.3",
+    "@types/leaflet": "1.9.0",
+    "@types/lodash-es": "4.17.7",
+    "typescript": "5.8.3"
+  }
+}

packages/shared-metrics/test/dependencies/test.js

@@ -212,6 +212,53 @@ describe('dependencies', function () {
         })
       ));
   });
+
+  describe('with many dependencies', () => {
+    const appDir = path.join(__dirname, 'app-with-many-dependencies');
+
+    let controls;
+
+    before(async () => {
+      // eslint-disable-next-line no-console
+      console.log('Installing dependencies for app-with-many-dependencies. This may take a while...');
+      runCommandSync('rm -rf node_modules', appDir);
+      runCommandSync('npm install --no-optional --no-audit --no-package-lock', appDir);
+      // eslint-disable-next-line no-console
+      console.log('Installed dependencies for app-with-many-dependencies');
+      controls = new ProcessControls({
+        dirname: appDir,
+        useGlobalAgent: true
+      });
+
+      await controls.startAndWaitForAgentConnection();
+    });
+
+    before(async () => {
+      await agentControls.clearReceivedTraceData();
+    });
+
+    after(async () => {
+      await controls.stop();
+    });
+
+    it('should limit dependencies', () => {
+      return retry(async () => {
+        const allMetrics = await agentControls.getAllMetrics(controls.getPid());
+        expect(allMetrics).to.be.an('array');
+
+        const deps = findMetric(allMetrics, ['dependencies']);
+        expect(deps).to.be.an('object');
+        expect(Object.keys(deps).length).to.be.at.least(500);
+        expect(Object.keys(deps).length).to.be.at.most(1000);
+
+        // expect that the first dependency is in the list
+        expect(deps['@ampproject/remapping']).to.exist;
+
+        // expect that the last dependency is in the list
+        expect(deps['zone.js']).to.exist;
+      });
+    });
+  });
 });
 
 /**