dependencies.js

/*
 * (c) Copyright IBM Corp. 2021
 * (c) Copyright Instana Inc. and contributors 2015
 */

'use strict';

const path = require('path');
const { util, uninstrumentedFs: fs } = require('@instana/core');

const CountDownLatch = require('./util/CountDownLatch');
const { DependencyDistanceCalculator, MAX_DEPTH } = require('./util/DependencyDistanceCalculator');

/** @type {import('@instana/core/src/core').GenericLogger} */
let logger;

/**
 * @param {import('@instana/core/src/config').InstanaConfig} config
 */
exports.init = function init(config) {
  logger = config.logger;
};

/** @type {number} */
exports.MAX_DEPENDENCIES = 750;

/** @type {string} */
exports.payloadPrefix = 'dependencies';

const MAX_DEPTH_NODE_MODULES = 2;

/** @type {Object.<string, string>} */
const preliminaryPayload = {};

/** @type {Object.<string, string>} */
// @ts-ignore: Cannot redeclare exported variable 'currentPayload'
exports.currentPayload = {};

// @ts-ignore
exports.activate = function activate(config, packageJsonObj) {
  const started = Date.now();

  if (!packageJsonObj || !packageJsonObj.file) {
    util.applicationUnderMonitoring.findNodeModulesFolder((errNodeModules, nodeModulesFolder) => {
      if (errNodeModules) {
        return logger.warn(
          `Failed to determine node_modules folder. Reason: ${errNodeModules?.message}, ${errNodeModules?.stack}`
        );
      } else if (!nodeModulesFolder) {
        return logger.warn(
          'Neither the package.json file nor the node_modules folder could be found. Stopping dependency analysis.'
        );
      }

      addAllDependencies(path.join(nodeModulesFolder), started, null);
    });

    return;
  }

  let dependencyDir;

  if (util.applicationUnderMonitoring.isAppInstalledIntoNodeModules()) {
    dependencyDir = path.join(path.dirname(packageJsonObj.path), '..', '..', 'node_modules');
  } else {
    dependencyDir = path.join(path.dirname(packageJsonObj.path), 'node_modules');
  }

  addAllDependencies(dependencyDir, started, packageJsonObj.path);
};

/**
 * Finds all installed modules in the given dependencyDir (say, /path/to/app/node_modules) and saves the dependency with
 * the associated version into preliminaryPayload.
 *
 * @param {string} dependencyDir
 * @param {number} started
 * @param {string} packageJsonPath
 */
function addAllDependencies(dependencyDir, started, packageJsonPath) {
  addDependenciesFromDir(dependencyDir, 0, () => {
    // TODO: This check happens AFTER we have already collected the dependencies.
    //       This is quiet useless for a large dependency tree, because we consume resources to collect
    //       all the dependencies (fs.stats, fs.readFile etc), but then discard most of them here.
    //       This is only critical for a very large number of defined dependencies in package.json (vertical).
    // NOTE: There is an extra protection in the `addDependenciesFromDir` fn to
    //       limit the depth of traversing node_modules.
    if (Object.keys(preliminaryPayload).length <= exports.MAX_DEPENDENCIES) {
      // @ts-ignore: Cannot redeclare exported variable 'currentPayload'
      exports.currentPayload = preliminaryPayload;
      logger.debug(`Collection of dependencies took ${Date.now() - started} ms.`);
      return;
    }

    if (packageJsonPath) {
      new DependencyDistanceCalculator().calculateDistancesFrom(packageJsonPath, distancesFromRoot => {
        logger.debug(`Collection of dependencies took ${Date.now() - started} ms.`);
        limitAndSet(distancesFromRoot);
      });
    } else {
      logger.debug(`Collection of dependencies took ${Date.now() - started} ms.`);
      limitAndSet();
    }
  });
}

/**
 * Finds all installed modules in dependencyDir (say, /path/to/app/node_modules) and saves the dependency with the
 * associated version into preliminaryPayload.
 *
 * @param {string} dependencyDir
 * @param {() => void} callback
 */
function addDependenciesFromDir(dependencyDir, currentDepth = 0, callback) {
  if (currentDepth >= MAX_DEPTH_NODE_MODULES) {
    return callback();
  }

  fs.readdir(dependencyDir, (readDirErr, dependencies) => {
    if (readDirErr || !dependencies) {
      logger.warn(`Cannot analyse dependencies due to ${readDirErr?.message}`);
      callback();
      return;
    }

    const filteredDependendencies = dependencies.filter(
      (
        dependency // exclude the .bin directory
      ) => dependency !== '.bin'
    );
    if (filteredDependendencies.length === 0) {
      callback();
      return;
    }

    // This latch fires once all dependencies of the current directory in the node_modules tree have been analysed.
    const countDownLatch = new CountDownLatch(filteredDependendencies.length);
    countDownLatch.once('done', () => {
      callback();
    });

    filteredDependendencies.forEach(dependency => {
      if (dependency.indexOf('@') === 0) {
        // NOTE: We do not increase currentDepth because scoped packages are just a folder containing more packages.
        addDependenciesFromDir(path.join(dependencyDir, dependency), currentDepth, () => {
          countDownLatch.countDown();
        });
      } else {
        const fullDirPath = path.join(dependencyDir, dependency);

        // Only check directories. For example, yarn adds a .yarn-integrity file to /node_modules/ which we need to
        // exclude, otherwise we get a confusing "Failed to identify version of .yarn-integrity dependency due to:
        // ENOTDIR: not a directory, open '.../node_modules/.yarn-integrity/package.json'." in the logs.
        fs.stat(fullDirPath, (statErr, stats) => {
          if (statErr) {
            countDownLatch.countDown();
            logger.warn(`Cannot analyse dependency ${fullDirPath} due to ${statErr?.message}`);
            return;
          }
          if (!stats.isDirectory()) {
            countDownLatch.countDown();
            return;
          }

          addDependency(dependency, fullDirPath, countDownLatch, currentDepth);
        });
      }
    });
  });
}

/**
 * Parses the package.json file in the given directory and then adds the given dependency (with its version) to
 * preliminaryPayload.
 *
 * @param {string} dependency
 * @param {string} dependencyDirPath
 * @param {import('./util/CountDownLatch')} countDownLatch
 * @param {number} currentDepth
 */
function addDependency(dependency, dependencyDirPath, countDownLatch, currentDepth) {
  const packageJsonPath = path.join(dependencyDirPath, 'package.json');

  fs.readFile(packageJsonPath, { encoding: 'utf8' }, (err, contents) => {
    if (err && err.code === 'ENOENT') {
      // This directory does not contain a package json. This happens for example for node_modules/.cache etc.
      // We can simply ignore this.
      countDownLatch.countDown();
      logger.debug(`No package.json at ${packageJsonPath}, ignoring this directory.`);
      return;
    } else if (err) {
      countDownLatch.countDown();
      logger.info(
        `Failed to identify version of ${dependency} dependency due to: ${err?.message}. ` +
          'This means that you will not be able to see details about this dependency within Instana.'
      );
      return;
    }

    try {
      const parsedPackageJson = JSON.parse(contents);
      if (!preliminaryPayload[parsedPackageJson.name]) {
        preliminaryPayload[parsedPackageJson.name] = parsedPackageJson.version;
      }
    } catch (parseErr) {
      // TODO: countDownLatch.countDown(); needs to be called here too?
      // countDownLatch.countDown();
      return logger.info(
        `Failed to identify version of ${dependency} dependency due to: ${parseErr?.message}.
          This means that you will not be able to see details about this dependency within Instana.`
      );
    }

    // NOTE: The dependency metric collector does not respect if the node_modules are dev dependencies or production
    //       dependencies. It collects all dependencies that are installed in the node_modules folder.
    const potentialNestedNodeModulesFolder = path.join(dependencyDirPath, 'node_modules');
    fs.stat(potentialNestedNodeModulesFolder, (statErr, stats) => {
      if (statErr || !stats.isDirectory()) {
        countDownLatch.countDown();
        return;
      }
      addDependenciesFromDir(potentialNestedNodeModulesFolder, currentDepth + 1, () => {
        countDownLatch.countDown();
      });
    });
  });
}

/**
 * Limits the collected dependencies to exports.MAX_DEPENDENCIES entries and commits them to exports.currentPayload.
 *
 * @param {Object<string, any>} distances
 */
function limitAndSet(distances = {}) {
  const keys = Object.keys(preliminaryPayload);
  keys.sort(sortByDistance.bind(null, distances));

  // After sorting, the most distant (and therefore, most uninteresting) packages are a the start of the array. For
  // packages with the same distance, we sort in a reverse lexicographic order. That means, that if no distances are
  // available at all, packages will be in reverse lexicographical order.
  //
  // At any rate, we start deleting collected depenencies from the payload at index 0, that is, we either remove the
  // most distant ones or the ones that are at the end of the lexicographic order.
  for (let i = 0; i < keys.length - exports.MAX_DEPENDENCIES; i++) {
    delete preliminaryPayload[keys[i]];
  }

  // @ts-ignore: Cannot redeclare exported variable 'currentPayload'
  exports.currentPayload = preliminaryPayload;
}

/**
 * Compares the given dependencies by their distance.
 *
 * @param {Object<string, any>} distances
 * @param {string} dependency1
 * @param {string} dependency2
 */
function sortByDistance(distances, dependency1, dependency2) {
  // To make troubleshooting easier, we always want to include the Instana dependencies, therefore they will be sorted
  // to the end of the array.
  const isInstana1 = dependency1.indexOf('instana') >= 0;
  const isInstana2 = dependency2.indexOf('instana') >= 0;
  if (isInstana1 && isInstana2) {
    return dependency2.localeCompare(dependency1);
  } else if (isInstana1) {
    return 1;
  } else if (isInstana2) {
    return -1;
  }

  const d1 = distances[dependency1] || MAX_DEPTH + 1;
  const d2 = distances[dependency2] || MAX_DEPTH + 1;
  if (d1 === d2) {
    // for the same distance, sort lexicographically
    return dependency2.localeCompare(dependency1);
  }
  return d2 - d1;
}