Support yarn dependency tree

Signed-off-by: 석지영/책임연구원/SW공학(연)Open Source TP <[email protected]>

Author: dd-jy

src/fosslight_dependency/_analyze_dependency.py

@@ -8,6 +8,7 @@
 import fosslight_dependency.constant as const
 from fosslight_dependency.package_manager.Pypi import Pypi
 from fosslight_dependency.package_manager.Npm import Npm
+from fosslight_dependency.package_manager.Yarn import Yarn
 from fosslight_dependency.package_manager.Maven import Maven
 from fosslight_dependency.package_manager.Gradle import Gradle
 from fosslight_dependency.package_manager.Pub import Pub
@@ -35,8 +36,10 @@ def analyze_dependency(package_manager_name, input_dir, output_dir, pip_activate
 
     if package_manager_name == const.PYPI:
         package_manager = Pypi(input_dir, output_dir, pip_activate_cmd, pip_deactivate_cmd)
-    elif package_manager_name == const.NPM or package_manager_name == const.YARN:
+    elif package_manager_name == const.NPM:
         package_manager = Npm(input_dir, output_dir)
+    elif package_manager_name == const.YARN:
+        package_manager = Yarn(input_dir, output_dir)
     elif package_manager_name == const.MAVEN:
         package_manager = Maven(input_dir, output_dir, output_custom_dir)
     elif package_manager_name == const.GRADLE:

src/fosslight_dependency/constant.py

@@ -32,6 +32,7 @@
     PYPI: ['requirements.txt', 'setup.py', 'pyproject.toml'],
     PNPM: 'pnpm-lock.yaml',
     NPM: 'package.json',
+    YARN: 'yarn.lock',
     MAVEN: 'pom.xml',
     GRADLE: 'build.gradle',
     PUB: 'pubspec.yaml',

src/fosslight_dependency/package_manager/Maven.py

@@ -46,12 +46,13 @@ def run_plugin(self):
         ret = True
 
         if not os.path.isfile(self.input_file_name):
-            self.is_run_plugin = True
             pom_backup = 'pom.xml_backup'
 
             ret = self.add_plugin_in_pom(pom_backup)
             if ret:
-                self.run_maven_plugin()
+                ret_plugin = self.run_maven_plugin()
+                if ret_plugin:
+                    self.is_run_plugin = True
 
             if os.path.isfile(pom_backup):
                 shutil.move(pom_backup, const.SUPPORT_PACKAE.get(self.package_manager_name))
@@ -133,6 +134,7 @@ def clean_run_maven_plugin_output(self):
             shutil.rmtree(top_path)
 
     def run_maven_plugin(self):
+        ret_plugin = True
         logger.info('Run maven license scanning plugin with temporary pom.xml')
         current_mode = ''
         if os.path.isfile('mvnw') or os.path.isfile('mvnw.cmd'):
@@ -148,21 +150,24 @@ def run_maven_plugin(self):
         ret = subprocess.call(cmd, shell=True)
         if ret != 0:
             logger.error(f"Failed to run maven plugin: {cmd}")
+            ret_plugin = False
 
-        cmd = f"{cmd_mvn} dependency:tree"
-        try:
-            ret_txt = subprocess.check_output(cmd, text=True, shell=True)
-            if ret_txt is not None:
-                self.parse_dependency_tree(ret_txt)
-                self.set_direct_dependencies(True)
-            else:
-                logger.error(f"Failed to run: {cmd}")
+        if ret_plugin:
+            cmd = f"{cmd_mvn} dependency:tree"
+            try:
+                ret_txt = subprocess.check_output(cmd, text=True, shell=True)
+                if ret_txt is not None:
+                    self.parse_dependency_tree(ret_txt)
+                    self.set_direct_dependencies(True)
+                else:
+                    logger.error(f"Failed to run: {cmd}")
+                    self.set_direct_dependencies(False)
+            except Exception as e:
+                logger.error(f"Failed to run '{cmd}': {e}")
                 self.set_direct_dependencies(False)
-        except Exception as e:
-            logger.error(f"Failed to run '{cmd}': {e}")
-            self.set_direct_dependencies(False)
         if current_mode:
             change_file_mode(cmd_mvn, current_mode)
+        return ret_plugin
 
     def create_dep_stack(self, dep_line):
         dep_stack = []

src/fosslight_dependency/package_manager/Npm.py

@@ -115,13 +115,15 @@ def parse_transitive_relationship(self):
         cmd = 'npm ls -a --omit=dev --json -s'
         result = subprocess.run(cmd, shell=True, capture_output=True, text=True, encoding='utf-8')
         rel_tree = result.stdout
-        if rel_tree is None:
-            logger.error(f"It returns the error: {cmd}")
-            logger.error(f"No output for {cmd}")
+        if not rel_tree or rel_tree.strip() == '':
+            logger.error(f"No output for {cmd}, stderr: {result.stderr}")
+            ret = False
+        elif result.returncode > 1:
+            logger.error(f"'{cmd}' failed with exit code({result.returncode}), stderr: {result.stderr}")
             ret = False
         if ret:
             if result.returncode == 1:
-                logger.warning(f"'{cmd}' returns error code: {result.stderr}")
+                logger.debug(f"'{cmd}' has warnings: {result.stderr}")
 
             try:
                 rel_json = json.loads(rel_tree)

src/fosslight_dependency/package_manager/Yarn.py

@@ -0,0 +1,224 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2025 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import logging
+import subprocess
+import json
+import fosslight_util.constant as constant
+import fosslight_dependency.constant as const
+from fosslight_dependency.package_manager.Npm import Npm
+from fosslight_dependency.dependency_item import DependencyItem, change_dependson_to_purl
+from fosslight_util.oss_item import OssItem
+from fosslight_dependency._package_manager import get_url_to_purl
+from fosslight_dependency.package_manager.Npm import check_multi_license, check_unknown_license
+
+logger = logging.getLogger(constant.LOGGER_NAME)
+
+
+class Yarn(Npm):
+    
+    def __init__(self, input_dir, output_dir):
+        super().__init__(input_dir, output_dir)
+        self.package_manager_name = const.YARN
+        self.yarn_version = None
+    
+    def detect_yarn_version(self):
+        """Detect Yarn version (1.x = Classic, 2+ = Berry)"""
+        if self.yarn_version is not None:
+            return self.yarn_version
+            
+        try:
+            result = subprocess.run('yarn -v', shell=True, capture_output=True, text=True, encoding='utf-8')
+            if result.returncode == 0:
+                version_str = result.stdout.strip()
+                major_version = int(version_str.split('.')[0])
+                self.yarn_version = major_version
+                logger.info(f"Detected Yarn version: {version_str} (major: {major_version})")
+                return major_version
+        except Exception as e:
+            logger.warning(f"Failed to detect Yarn version: {e}")
+        return None
+
+    def start_license_checker(self):
+        ret = True
+        license_checker_cmd = f'license-checker --production --json --out {self.input_file_name}'
+        custom_path_option = ' --customPath '
+        node_modules = 'node_modules'
+        
+        # Detect Yarn version
+        self.detect_yarn_version()
+        
+        # For Yarn Berry (2+), check if using PnP mode
+        is_pnp_mode = False
+        if self.yarn_version and self.yarn_version >= 2:
+            # Check if .pnp.cjs exists (PnP mode indicator)
+            if os.path.exists('.pnp.cjs') or os.path.exists('.pnp.js'):
+                is_pnp_mode = True
+                logger.info("Detected Yarn Berry with PnP mode")
+        
+        if not os.path.isdir(node_modules):
+            logger.info(f"node_modules directory does not exist.")
+            self.flag_tmp_node_modules = True
+            
+            # For PnP mode, try to force node_modules creation
+            if is_pnp_mode:
+                logger.info("Attempting to create node_modules for PnP project...")
+                # Try setting nodeLinker to node-modules temporarily
+                yarn_install_cmd = 'YARN_NODE_LINKER=node-modules yarn install --production --ignore-scripts'
+                logger.info(f"Executing: {yarn_install_cmd}")
+            else:
+                yarn_install_cmd = 'yarn install --production --ignore-scripts'
+                logger.info(f"Executing: {yarn_install_cmd}")
+            
+            cmd_ret = subprocess.call(yarn_install_cmd, shell=True)
+            if cmd_ret != 0:
+                logger.error(f"{yarn_install_cmd} failed")
+                if is_pnp_mode:
+                    logger.error("Yarn Berry PnP mode detected. Consider setting 'nodeLinker: node-modules' in .yarnrc.yml")
+                return False
+            else:
+                logger.info(f"Successfully executed {yarn_install_cmd}")
+
+        self.make_custom_json(self.tmp_custom_json)
+
+        cmd = license_checker_cmd + custom_path_option + self.tmp_custom_json
+        cmd_ret = subprocess.call(cmd, shell=True)
+        if cmd_ret != 0:
+            logger.error(f"It returns the error: {cmd}")
+            logger.error("Please check if the license-checker is installed.(sudo npm install -g license-checker)")
+            ret = False
+        else:
+            self.append_input_package_list_file(self.input_file_name)
+        if os.path.exists(self.tmp_custom_json):
+            os.remove(self.tmp_custom_json)
+
+        return ret
+
+    def parse_oss_information(self, f_name):
+        with open(f_name, 'r', encoding='utf8') as json_file:
+            json_data = json.load(json_file)
+
+        _licenses = 'licenses'
+        _repository = 'repository'
+        _private = 'private'
+
+        keys = [key for key in json_data]
+        purl_dict = {}
+        for i in range(0, len(keys)):
+            dep_item = DependencyItem()
+            oss_item = OssItem()
+            d = json_data.get(keys[i - 1])
+            oss_init_name = d['name']
+            oss_item.name = f'{const.NPM}:{oss_init_name}'
+
+            if d[_licenses]:
+                license_name = d[_licenses]
+            else:
+                license_name = ''
+
+            oss_item.version = d['version']
+            package_path = d['path']
+
+            private_pkg = False
+            if _private in d:
+                if d[_private]:
+                    private_pkg = True
+
+            oss_item.download_location = f"{self.dn_url}{oss_init_name}/v/{oss_item.version}"
+            dn_loc = f"{self.dn_url}{oss_init_name}"
+            dep_item.purl = get_url_to_purl(oss_item.download_location, self.package_manager_name)
+            purl_dict[f'{oss_init_name}({oss_item.version})'] = dep_item.purl
+            if d[_repository]:
+                dn_loc = d[_repository]
+            elif private_pkg:
+                dn_loc = ''
+
+            oss_item.homepage = dn_loc
+
+            if private_pkg:
+                oss_item.download_location = oss_item.homepage
+                oss_item.comment = 'private'
+            if self.package_name == f'{oss_init_name}({oss_item.version})':
+                oss_item.comment = 'root package'
+            elif self.direct_dep and len(self.relation_tree) > 0:
+                if f'{oss_init_name}({oss_item.version})' in self.relation_tree[self.package_name]:
+                    oss_item.comment = 'direct'
+                else:
+                    oss_item.comment = 'transitive'
+
+                if f'{oss_init_name}({oss_item.version})' in self.relation_tree:
+                    dep_item.depends_on_raw = self.relation_tree[f'{oss_init_name}({oss_item.version})']
+
+            # For Yarn, use 'package.json' instead of yarn.lock for license info
+            manifest_file_path = os.path.join(package_path, 'package.json')
+            multi_license, license_comment, multi_flag = check_multi_license(license_name, manifest_file_path)
+
+            if multi_flag:
+                oss_item.comment = license_comment
+                license_name = multi_license
+            else:
+                license_name = license_name.replace(",", "")
+                license_name = check_unknown_license(license_name, manifest_file_path)
+            oss_item.license = license_name
+
+            dep_item.oss_items.append(oss_item)
+            self.dep_items.append(dep_item)
+
+        if self.direct_dep:
+            self.dep_items = change_dependson_to_purl(purl_dict, self.dep_items)
+        return
+
+    def parse_rel_dependencies(self, rel_name, rel_ver, rel_dependencies):
+        """Override to handle missing packages and packages without version"""
+        _dependencies = 'dependencies'
+        _version = 'version'
+        _peer = 'peerMissing'
+        _missing = 'missing'
+
+        for rel_dep_name in rel_dependencies.keys():
+            # Optional, non-installed dependencies are listed as empty objects
+            if rel_dependencies[rel_dep_name] == {}:
+                continue
+            if _peer in rel_dependencies[rel_dep_name]:
+                if rel_dependencies[rel_dep_name][_peer]:
+                    continue
+            # Skip missing packages (not installed)
+            if _missing in rel_dependencies[rel_dep_name]:
+                if rel_dependencies[rel_dep_name][_missing]:
+                    continue
+            # Skip if version key doesn't exist
+            if _version not in rel_dependencies[rel_dep_name]:
+                continue
+
+            if f'{rel_name}({rel_ver})' not in self.relation_tree:
+                self.relation_tree[f'{rel_name}({rel_ver})'] = []
+            elif f'{rel_dep_name}({rel_dependencies[rel_dep_name][_version]})' in self.relation_tree[f'{rel_name}({rel_ver})']:
+                continue
+            self.relation_tree[f'{rel_name}({rel_ver})'].append(f'{rel_dep_name}({rel_dependencies[rel_dep_name][_version]})')
+            if _dependencies in rel_dependencies[rel_dep_name]:
+                self.parse_rel_dependencies(rel_dep_name, rel_dependencies[rel_dep_name][_version],
+                                            rel_dependencies[rel_dep_name][_dependencies])
+
+    def parse_direct_dependencies(self):
+        if not self.direct_dep:
+            return
+        try:
+            # For Yarn, check if package.json exists (not yarn.lock)
+            # input_package_list_file[0] is the license-checker output file path
+            manifest_dir = os.path.dirname(self.input_package_list_file[0])
+            package_json_path = os.path.join(manifest_dir, 'package.json')
+
+            if os.path.isfile(package_json_path):
+                ret, err_msg = self.parse_transitive_relationship()
+                if not ret:
+                    self.direct_dep = False
+                    logger.warning(f'It cannot print direct/transitive dependency: {err_msg}')
+            else:
+                logger.info('Direct/transitive support is not possible because the package.json file does not exist.')
+                self.direct_dep = False
+        except Exception as e:
+            logger.warning(f'Cannot print direct/transitive dependency: {e}')
+            self.direct_dep = False

src/fosslight_dependency/run_dependency_scanner.py

@@ -106,6 +106,22 @@ def find_package_manager(input_dir, abs_path_to_exclude=[], manifest_file_name=[
     # both npm and pnpm are detected, remove npm.
     if 'npm' in found_package_manager.keys() and 'pnpm' in found_package_manager.keys():
         del found_package_manager['npm']
+    
+    # both npm and yarn are detected, check which one to use based on lock file
+    if 'npm' in found_package_manager.keys() and 'yarn' in found_package_manager.keys():
+        # Remove npm from directories where yarn.lock exists
+        dirs_to_remove_from_npm = []
+        for yarn_dir in found_package_manager['yarn'].keys():
+            if yarn_dir in found_package_manager['npm']:
+                dirs_to_remove_from_npm.append(yarn_dir)
+        
+        for dir_to_remove in dirs_to_remove_from_npm:
+            del found_package_manager['npm'][dir_to_remove]
+        
+        # If npm has no directories left, remove it entirely
+        if not found_package_manager['npm']:
+            del found_package_manager['npm']
+    
     if len(found_package_manager) >= 1:
         log_lines = ["\nDetected Manifest Files automatically"]
         log_lines = print_package_info(found_package_manager, log_lines)
@@ -208,8 +224,6 @@ def run_dependency_scanner(package_manager='', input_dir='', output_dir_file='',
     found_package_manager = {}
     if package_manager:
         scan_item.set_cover_comment(f"Manual detect mode (-m {package_manager})")
-        if package_manager == const.YARN:
-            package_manager = const.NPM
         autodetect = False
         support_packagemanager = list(const.SUPPORT_PACKAE.keys())