diff --git a/src/fosslight_dependency/_analyze_dependency.py b/src/fosslight_dependency/_analyze_dependency.py index 64bd1028..23c35938 100644 --- a/src/fosslight_dependency/_analyze_dependency.py +++ b/src/fosslight_dependency/_analyze_dependency.py @@ -8,6 +8,7 @@ import fosslight_dependency.constant as const from fosslight_dependency.package_manager.Pypi import Pypi from fosslight_dependency.package_manager.Npm import Npm +from fosslight_dependency.package_manager.Yarn import Yarn from fosslight_dependency.package_manager.Maven import Maven from fosslight_dependency.package_manager.Gradle import Gradle from fosslight_dependency.package_manager.Pub import Pub @@ -32,11 +33,15 @@ def analyze_dependency(package_manager_name, input_dir, output_dir, pip_activate ret = True package_dep_item_list = [] cover_comment = '' + npm_fallback_to_yarn = False if package_manager_name == const.PYPI: package_manager = Pypi(input_dir, output_dir, pip_activate_cmd, pip_deactivate_cmd) - elif package_manager_name == const.NPM or package_manager_name == const.YARN: + elif package_manager_name == const.NPM: package_manager = Npm(input_dir, output_dir) + npm_fallback_to_yarn = True + elif package_manager_name == const.YARN: + package_manager = Yarn(input_dir, output_dir) elif package_manager_name == const.MAVEN: package_manager = Maven(input_dir, output_dir, output_custom_dir) elif package_manager_name == const.GRADLE: @@ -66,7 +71,7 @@ def analyze_dependency(package_manager_name, input_dir, output_dir, pip_activate else: logger.error(f"Not supported package manager name: {package_manager_name}") ret = False - return ret, package_dep_item_list + return ret, package_dep_item_list, cover_comment, package_manager_name if manifest_file_name: package_manager.set_manifest_file(manifest_file_name) @@ -74,6 +79,24 @@ def analyze_dependency(package_manager_name, input_dir, output_dir, pip_activate if direct: package_manager.set_direct_dependencies(direct) ret = package_manager.run_plugin() + + if not ret and npm_fallback_to_yarn: + logger.warning("Npm analysis failed. Attempting to use Yarn as fallback...") + del package_manager + package_manager = Yarn(input_dir, output_dir) + package_manager_name = const.YARN + + if manifest_file_name: + package_manager.set_manifest_file(manifest_file_name) + if direct: + package_manager.set_direct_dependencies(direct) + + ret = package_manager.run_plugin() + if ret: + logger.info("Successfully switched to Yarn") + else: + logger.error("Yarn also failed") + if ret: if direct: package_manager.parse_direct_dependencies() @@ -100,4 +123,4 @@ def analyze_dependency(package_manager_name, input_dir, output_dir, pip_activate del package_manager - return ret, package_dep_item_list, cover_comment + return ret, package_dep_item_list, cover_comment, package_manager_name diff --git a/src/fosslight_dependency/constant.py b/src/fosslight_dependency/constant.py index e0e1b1b4..9df9758d 100644 --- a/src/fosslight_dependency/constant.py +++ b/src/fosslight_dependency/constant.py @@ -32,6 +32,7 @@ PYPI: ['requirements.txt', 'setup.py', 'pyproject.toml'], PNPM: 'pnpm-lock.yaml', NPM: 'package.json', + YARN: 'yarn.lock', MAVEN: 'pom.xml', GRADLE: 'build.gradle', PUB: 'pubspec.yaml', diff --git a/src/fosslight_dependency/package_manager/Maven.py b/src/fosslight_dependency/package_manager/Maven.py index a3fe9397..fefd2686 100644 --- a/src/fosslight_dependency/package_manager/Maven.py +++ b/src/fosslight_dependency/package_manager/Maven.py @@ -46,12 +46,13 @@ def run_plugin(self): ret = True if not os.path.isfile(self.input_file_name): - self.is_run_plugin = True pom_backup = 'pom.xml_backup' ret = self.add_plugin_in_pom(pom_backup) if ret: - self.run_maven_plugin() + ret_plugin = self.run_maven_plugin() + if ret_plugin: + self.is_run_plugin = True if os.path.isfile(pom_backup): shutil.move(pom_backup, const.SUPPORT_PACKAE.get(self.package_manager_name)) @@ -133,6 +134,7 @@ def clean_run_maven_plugin_output(self): shutil.rmtree(top_path) def run_maven_plugin(self): + ret_plugin = True logger.info('Run maven license scanning plugin with temporary pom.xml') current_mode = '' if os.path.isfile('mvnw') or os.path.isfile('mvnw.cmd'): @@ -148,21 +150,24 @@ def run_maven_plugin(self): ret = subprocess.call(cmd, shell=True) if ret != 0: logger.error(f"Failed to run maven plugin: {cmd}") + ret_plugin = False - cmd = f"{cmd_mvn} dependency:tree" - try: - ret_txt = subprocess.check_output(cmd, text=True, shell=True) - if ret_txt is not None: - self.parse_dependency_tree(ret_txt) - self.set_direct_dependencies(True) - else: - logger.error(f"Failed to run: {cmd}") + if ret_plugin: + cmd = f"{cmd_mvn} dependency:tree" + try: + ret_txt = subprocess.check_output(cmd, text=True, shell=True) + if ret_txt is not None: + self.parse_dependency_tree(ret_txt) + self.set_direct_dependencies(True) + else: + logger.error(f"Failed to run: {cmd}") + self.set_direct_dependencies(False) + except Exception as e: + logger.error(f"Failed to run '{cmd}': {e}") self.set_direct_dependencies(False) - except Exception as e: - logger.error(f"Failed to run '{cmd}': {e}") - self.set_direct_dependencies(False) if current_mode: change_file_mode(cmd_mvn, current_mode) + return ret_plugin def create_dep_stack(self, dep_line): dep_stack = [] diff --git a/src/fosslight_dependency/package_manager/Npm.py b/src/fosslight_dependency/package_manager/Npm.py index 60b60a60..4ba64335 100644 --- a/src/fosslight_dependency/package_manager/Npm.py +++ b/src/fosslight_dependency/package_manager/Npm.py @@ -53,14 +53,8 @@ def start_license_checker(self): self.flag_tmp_node_modules = True cmd_ret = subprocess.call(npm_install_cmd, shell=True) if cmd_ret != 0: - logger.warning(f"{npm_install_cmd} returns an error. Trying yarn as fallback...") - yarn_install_cmd = 'yarn install --production --ignore-scripts' - cmd_ret = subprocess.call(yarn_install_cmd, shell=True) - if cmd_ret != 0: - logger.error(f"Both {npm_install_cmd} and {yarn_install_cmd} failed") - return False - else: - logger.info(f"Successfully executed {yarn_install_cmd}") + logger.error(f"{npm_install_cmd} failed") + return False # customized json file for obtaining specific items with license-checker self.make_custom_json(self.tmp_custom_json) @@ -115,13 +109,15 @@ def parse_transitive_relationship(self): cmd = 'npm ls -a --omit=dev --json -s' result = subprocess.run(cmd, shell=True, capture_output=True, text=True, encoding='utf-8') rel_tree = result.stdout - if rel_tree is None: - logger.error(f"It returns the error: {cmd}") - logger.error(f"No output for {cmd}") + if not rel_tree or rel_tree.strip() == '': + logger.error(f"No output for {cmd}, stderr: {result.stderr}") + ret = False + elif result.returncode > 1: + logger.error(f"'{cmd}' failed with exit code({result.returncode}), stderr: {result.stderr}") ret = False if ret: if result.returncode == 1: - logger.warning(f"'{cmd}' returns error code: {result.stderr}") + logger.debug(f"'{cmd}' has warnings: {result.stderr}") try: rel_json = json.loads(rel_tree) diff --git a/src/fosslight_dependency/package_manager/Yarn.py b/src/fosslight_dependency/package_manager/Yarn.py new file mode 100644 index 00000000..0434d752 --- /dev/null +++ b/src/fosslight_dependency/package_manager/Yarn.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2025 LG Electronics Inc. +# SPDX-License-Identifier: Apache-2.0 + +import os +import logging +import subprocess +import json +import fosslight_util.constant as constant +import fosslight_dependency.constant as const +from fosslight_dependency.package_manager.Npm import Npm +from fosslight_dependency.dependency_item import DependencyItem, change_dependson_to_purl +from fosslight_util.oss_item import OssItem +from fosslight_dependency._package_manager import get_url_to_purl +from fosslight_dependency.package_manager.Npm import check_multi_license, check_unknown_license + +logger = logging.getLogger(constant.LOGGER_NAME) + + +class Yarn(Npm): + + def __init__(self, input_dir, output_dir): + super().__init__(input_dir, output_dir) + self.package_manager_name = const.YARN + self.yarn_version = None + + def detect_yarn_version(self): + """Detect Yarn version (1.x = Classic, 2+ = Berry)""" + if self.yarn_version is not None: + return self.yarn_version + + try: + result = subprocess.run('yarn -v', shell=True, capture_output=True, text=True, encoding='utf-8') + if result.returncode == 0: + version_str = result.stdout.strip() + major_version = int(version_str.split('.')[0]) + self.yarn_version = major_version + logger.info(f"Detected Yarn version: {version_str} (major: {major_version})") + return major_version + except Exception as e: + logger.warning(f"Failed to detect Yarn version: {e}") + return None + + def start_license_checker(self): + ret = True + license_checker_cmd = f'license-checker --production --json --out {self.input_file_name}' + custom_path_option = ' --customPath ' + node_modules = 'node_modules' + + self.detect_yarn_version() + + # For Yarn Berry (2+), check if using PnP mode + is_pnp_mode = False + if self.yarn_version and self.yarn_version >= 2: + # Check if .pnp.cjs exists (PnP mode indicator) + if os.path.exists('.pnp.cjs') or os.path.exists('.pnp.js'): + is_pnp_mode = True + logger.info("Detected Yarn Berry with PnP mode") + + if not os.path.isdir(node_modules): + logger.info("node_modules directory does not exist.") + self.flag_tmp_node_modules = True + + # For PnP mode, try to force node_modules creation + if is_pnp_mode: + logger.info("Attempting to create node_modules for PnP project...") + yarn_install_cmd = 'YARN_NODE_LINKER=node-modules yarn install --production --ignore-scripts' + logger.info(f"Executing: {yarn_install_cmd}") + else: + yarn_install_cmd = 'yarn install --production --ignore-scripts' + logger.info(f"Executing: {yarn_install_cmd}") + + cmd_ret = subprocess.call(yarn_install_cmd, shell=True) + if cmd_ret != 0: + logger.error(f"{yarn_install_cmd} failed") + if is_pnp_mode: + logger.error("Yarn Berry PnP mode detected. Consider setting 'nodeLinker: node-modules' in .yarnrc.yml") + return False + else: + logger.info(f"Successfully executed {yarn_install_cmd}") + + self.make_custom_json(self.tmp_custom_json) + + cmd = license_checker_cmd + custom_path_option + self.tmp_custom_json + cmd_ret = subprocess.call(cmd, shell=True) + if cmd_ret != 0: + logger.error(f"It returns the error: {cmd}") + logger.error("Please check if the license-checker is installed.(sudo npm install -g license-checker)") + ret = False + else: + self.append_input_package_list_file(self.input_file_name) + if os.path.exists(self.tmp_custom_json): + os.remove(self.tmp_custom_json) + + return ret + + def parse_oss_information(self, f_name): + with open(f_name, 'r', encoding='utf8') as json_file: + json_data = json.load(json_file) + + _licenses = 'licenses' + _repository = 'repository' + _private = 'private' + + keys = [key for key in json_data] + purl_dict = {} + for i in range(0, len(keys)): + dep_item = DependencyItem() + oss_item = OssItem() + d = json_data.get(keys[i - 1]) + oss_init_name = d['name'] + oss_item.name = f'{const.NPM}:{oss_init_name}' + + if d[_licenses]: + license_name = d[_licenses] + else: + license_name = '' + + oss_item.version = d['version'] + package_path = d['path'] + + private_pkg = False + if _private in d: + if d[_private]: + private_pkg = True + + oss_item.download_location = f"{self.dn_url}{oss_init_name}/v/{oss_item.version}" + dn_loc = f"{self.dn_url}{oss_init_name}" + dep_item.purl = get_url_to_purl(oss_item.download_location, self.package_manager_name) + purl_dict[f'{oss_init_name}({oss_item.version})'] = dep_item.purl + if d[_repository]: + dn_loc = d[_repository] + elif private_pkg: + dn_loc = '' + + oss_item.homepage = dn_loc + + if private_pkg: + oss_item.download_location = oss_item.homepage + oss_item.comment = 'private' + if self.package_name == f'{oss_init_name}({oss_item.version})': + oss_item.comment = 'root package' + elif self.direct_dep and len(self.relation_tree) > 0: + if f'{oss_init_name}({oss_item.version})' in self.relation_tree[self.package_name]: + oss_item.comment = 'direct' + else: + oss_item.comment = 'transitive' + + if f'{oss_init_name}({oss_item.version})' in self.relation_tree: + dep_item.depends_on_raw = self.relation_tree[f'{oss_init_name}({oss_item.version})'] + + # For Yarn, use 'package.json' instead of yarn.lock for license info + manifest_file_path = os.path.join(package_path, 'package.json') + multi_license, license_comment, multi_flag = check_multi_license(license_name, manifest_file_path) + + if multi_flag: + oss_item.comment = license_comment + license_name = multi_license + else: + license_name = license_name.replace(",", "") + license_name = check_unknown_license(license_name, manifest_file_path) + oss_item.license = license_name + + dep_item.oss_items.append(oss_item) + self.dep_items.append(dep_item) + + if self.direct_dep: + self.dep_items = change_dependson_to_purl(purl_dict, self.dep_items) + return + + def parse_rel_dependencies(self, rel_name, rel_ver, rel_dependencies): + """Override to handle missing packages and packages without version""" + _dependencies = 'dependencies' + _version = 'version' + _peer = 'peerMissing' + _missing = 'missing' + + for rel_dep_name in rel_dependencies.keys(): + # Optional, non-installed dependencies are listed as empty objects + if rel_dependencies[rel_dep_name] == {}: + continue + if _peer in rel_dependencies[rel_dep_name]: + if rel_dependencies[rel_dep_name][_peer]: + continue + # Skip missing packages (not installed) + if _missing in rel_dependencies[rel_dep_name]: + if rel_dependencies[rel_dep_name][_missing]: + continue + # Skip if version key doesn't exist + if _version not in rel_dependencies[rel_dep_name]: + continue + + if f'{rel_name}({rel_ver})' not in self.relation_tree: + self.relation_tree[f'{rel_name}({rel_ver})'] = [] + elif f'{rel_dep_name}({rel_dependencies[rel_dep_name][_version]})' in self.relation_tree[f'{rel_name}({rel_ver})']: + continue + self.relation_tree[f'{rel_name}({rel_ver})'].append(f'{rel_dep_name}({rel_dependencies[rel_dep_name][_version]})') + if _dependencies in rel_dependencies[rel_dep_name]: + self.parse_rel_dependencies(rel_dep_name, rel_dependencies[rel_dep_name][_version], + rel_dependencies[rel_dep_name][_dependencies]) + + def parse_direct_dependencies(self): + if not self.direct_dep: + return + try: + # For Yarn, check if package.json exists (not yarn.lock) + # input_package_list_file[0] is the license-checker output file path + manifest_dir = os.path.dirname(self.input_package_list_file[0]) + package_json_path = os.path.join(manifest_dir, 'package.json') + + if os.path.isfile(package_json_path): + ret, err_msg = self.parse_transitive_relationship() + if not ret: + self.direct_dep = False + logger.warning(f'It cannot print direct/transitive dependency: {err_msg}') + else: + logger.info('Direct/transitive support is not possible because the package.json file does not exist.') + self.direct_dep = False + except Exception as e: + logger.warning(f'Cannot print direct/transitive dependency: {e}') + self.direct_dep = False diff --git a/src/fosslight_dependency/run_dependency_scanner.py b/src/fosslight_dependency/run_dependency_scanner.py index 762bedd7..d0505d9f 100755 --- a/src/fosslight_dependency/run_dependency_scanner.py +++ b/src/fosslight_dependency/run_dependency_scanner.py @@ -106,6 +106,22 @@ def find_package_manager(input_dir, abs_path_to_exclude=[], manifest_file_name=[ # both npm and pnpm are detected, remove npm. if 'npm' in found_package_manager.keys() and 'pnpm' in found_package_manager.keys(): del found_package_manager['npm'] + + # both npm and yarn are detected, check which one to use based on lock file + if 'npm' in found_package_manager.keys() and 'yarn' in found_package_manager.keys(): + # Remove npm from directories where yarn.lock exists + dirs_to_remove_from_npm = [] + for yarn_dir in found_package_manager['yarn'].keys(): + if yarn_dir in found_package_manager['npm']: + dirs_to_remove_from_npm.append(yarn_dir) + + for dir_to_remove in dirs_to_remove_from_npm: + del found_package_manager['npm'][dir_to_remove] + + # If npm has no directories left, remove it entirely + if not found_package_manager['npm']: + del found_package_manager['npm'] + if len(found_package_manager) >= 1: log_lines = ["\nDetected Manifest Files automatically"] log_lines = print_package_info(found_package_manager, log_lines) @@ -208,8 +224,6 @@ def run_dependency_scanner(package_manager='', input_dir='', output_dir_file='', found_package_manager = {} if package_manager: scan_item.set_cover_comment(f"Manual detect mode (-m {package_manager})") - if package_manager == const.YARN: - package_manager = const.NPM autodetect = False support_packagemanager = list(const.SUPPORT_PACKAE.keys()) @@ -261,33 +275,34 @@ def run_dependency_scanner(package_manager='', input_dir='', output_dir_file='', cover_comment = '' for pm, manifest_file_name_list in found_package_manager.items(): if not manifest_file_name_list and not autodetect: - ret, package_dep_item_list, cover_comment = analyze_dependency(pm, input_dir, output_path, - pip_activate_cmd, pip_deactivate_cmd, - output_custom_dir, app_name, github_token, - [], direct) + ret, package_dep_item_list, cover_comment, actual_pm = analyze_dependency(pm, input_dir, output_path, + pip_activate_cmd, pip_deactivate_cmd, + output_custom_dir, app_name, github_token, + [], direct) if ret: - success_pm[pm][input_dir].extend(['manual mode (-m option)']) + success_pm[actual_pm][input_dir].extend(['manual mode (-m option)']) scan_item.append_file_items(package_dep_item_list) else: - fail_pm[pm][input_dir].extend(['manual mode (-m option)']) + fail_pm[actual_pm][input_dir].extend(['manual mode (-m option)']) else: for manifest_dir, manifest_file_name in manifest_file_name_list.items(): input_dir = manifest_dir if manifest_file_name == pass_key: continue os.chdir(input_dir) - ret, package_dep_item_list, cover_comment = analyze_dependency(pm, input_dir, output_path, - pip_activate_cmd, pip_deactivate_cmd, - output_custom_dir, app_name, github_token, - manifest_file_name, direct) + ret, package_dep_item_list, cover_comment, actual_pm = analyze_dependency(pm, input_dir, output_path, + pip_activate_cmd, pip_deactivate_cmd, + output_custom_dir, app_name, + github_token, + manifest_file_name, direct) if ret: - success_pm[pm][input_dir].extend(manifest_file_name) + success_pm[actual_pm][input_dir].extend(manifest_file_name) scan_item.append_file_items(package_dep_item_list) dup_pm = None - if pm == const.GRADLE and const.ANDROID in found_package_manager: + if actual_pm == const.GRADLE and const.ANDROID in found_package_manager: dup_pm = const.ANDROID - elif pm == const.ANDROID and const.GRADLE in found_package_manager: + elif actual_pm == const.ANDROID and const.GRADLE in found_package_manager: dup_pm = const.GRADLE if dup_pm: @@ -298,7 +313,7 @@ def run_dependency_scanner(package_manager='', input_dir='', output_dir_file='', else: found_package_manager[dup_pm][manifest_dir] = pass_key else: - fail_pm[pm][input_dir].extend(manifest_file_name) + fail_pm[actual_pm][input_dir].extend(manifest_file_name) success_pm = {k: dict(v) for k, v in success_pm.items()} fail_pm = {k: dict(v) for k, v in fail_pm.items()}