Print the dependencies of each package in comment (#128)

Signed-off-by: Jiyeong Seok <[email protected]>

Author: dd-jy

src/fosslight_dependency/_package_manager.py

@@ -11,7 +11,6 @@
 import base64
 import subprocess
 import shutil
-import copy
 import fosslight_util.constant as constant
 import fosslight_dependency.constant as const
 
@@ -47,6 +46,8 @@ def __init__(self, package_manager_name, dn_url, input_dir, output_dir):
         self.output_dir = output_dir
         self.dn_url = dn_url
         self.manifest_file_name = []
+        self.relation_tree = {}
+        self.package_name = ''
 
         self.platform = platform.system()
         self.license_scanner_bin = check_license_scanner(self.platform)
@@ -61,6 +62,8 @@ def __del__(self):
         self.output_dir = ''
         self.dn_url = ''
         self.manifest_file_name = []
+        self.relation_tree = {}
+        self.package_name = ''
 
     def run_plugin(self):
         if self.package_manager_name == const.GRADLE or self.package_manager_name == const.ANDROID:
@@ -87,23 +90,20 @@ def run_gradle_task(self):
 
             shutil.copy(const.SUPPORT_PACKAE.get(self.package_manager_name), gradle_backup)
             ret = self.add_allDeps_in_gradle()
-            if not ret:
-                return
-
-            if os.path.isfile('gradlew') or os.path.isfile('gradlew.bat'):
-                if self.platform == const.WINDOWS:
-                    cmd_gradle = "gradlew.bat"
-                else:
-                    cmd_gradle = "./gradlew"
-            else:
-                return 1
-            cmd = f"{cmd_gradle} allDeps"
-            ret = subprocess.check_output(cmd, shell=True, encoding='utf-8')
-            if ret != 0:
-                self.parse_dependency_tree(ret)
-            else:
-                self.set_direct_dependencies(False)
-                logger.warning("Failed to run allDeps task.")
+            if ret:
+                if os.path.isfile('gradlew') or os.path.isfile('gradlew.bat'):
+                    if self.platform == const.WINDOWS:
+                        cmd_gradle = "gradlew.bat"
+                    else:
+                        cmd_gradle = "./gradlew"
+
+                    cmd = f"{cmd_gradle} allDeps"
+                    ret = subprocess.check_output(cmd, shell=True, encoding='utf-8')
+                    if ret != 0:
+                        self.parse_dependency_tree(ret)
+                    else:
+                        self.set_direct_dependencies(False)
+                        logger.warning("Failed to run allDeps task.")
 
             if os.path.isfile(gradle_backup):
                 os.remove(const.SUPPORT_PACKAE.get(self.package_manager_name))
@@ -132,26 +132,55 @@ def add_allDeps_in_gradle(self):
 
         return ret
 
-    def parse_dependency_tree(self, dependency_tree_fname):
-        config = android_config if self.package_manager_name == 'android' else gradle_config
+    def create_dep_stack(self, dep_line, config):
         packages_in_config = False
-        for line in dependency_tree_fname.split('\n'):
+        dep_stack = []
+        cur_flag = ''
+        dep_level = -1
+        dep_level_plus = False
+        for line in dep_line.split('\n'):
             try:
-                line_bk = copy.deepcopy(line)
                 if not packages_in_config:
                     filtered = next(filter(lambda c: re.findall(rf'^{c}\s\-', line), config), None)
                     if filtered:
                         packages_in_config = True
                 else:
                     if line == '':
                         packages_in_config = False
-                    re_result = re.findall(r'\-\-\-\s([^\:\s]+\:[^\:\s]+)\:([^\:\s]+)', line)
+                    prev_flag = cur_flag
+                    prev_dep_level = dep_level
+                    dep_level = line.count("|")
+
+                    re_result = re.findall(r'([\+|\\])\-\-\-\s([^\:\s]+\:[^\:\s]+)\:([^\:\s]+)', line)
                     if re_result:
-                        self.total_dep_list.append(re_result[0][0])
-                        if re.match(r'^[\+|\\]\-\-\-\s([^\:\s]+\:[^\:\s]+)\:([^\:\s]+)', line_bk):
-                            self.direct_dep_list.append(re_result[0][0])
+                        cur_flag = re_result[0][0]
+                        if (prev_flag == '\\') and (prev_dep_level == dep_level):
+                            dep_level_plus = True
+                        if dep_level_plus and (prev_flag == '\\') and (prev_dep_level != dep_level):
+                            dep_level_plus = False
+                        if dep_level_plus:
+                            dep_level += 1
+                        dep_name = f'{re_result[0][1]}({re_result[0][2]})'
+                        dep_stack = dep_stack[:dep_level] + [dep_name]
+                        yield dep_stack[:dep_level], dep_name
+                    else:
+                        cur_flag = ''
             except Exception as e:
-                logger.error(f"Failed to parse dependency tree: {e}")
+                logger.warning(f"Failed to parse dependency tree: {e}")
+
+    def parse_dependency_tree(self, f_name):
+        config = android_config if self.package_manager_name == 'android' else gradle_config
+        try:
+            for stack, name in self.create_dep_stack(f_name, config):
+                self.total_dep_list.append(name)
+                if len(stack) == 0:
+                    self.direct_dep_list.append(name)
+                else:
+                    if stack[-1] not in self.relation_tree:
+                        self.relation_tree[stack[-1]] = []
+                    self.relation_tree[stack[-1]].append(name)
+        except Exception as e:
+            logger.warning(f'Fail to parse gradle dependency tree:{e}')
 
 
 def version_refine(oss_version):

src/fosslight_dependency/package_manager/Android.py

@@ -50,15 +50,22 @@ def parse_oss_information(self, f_name):
                 else:
                     continue
 
-                if self.total_dep_list:
-                    if oss_name not in self.total_dep_list:
-                        continue
-
+                comment_list = []
                 if self.direct_dep:
-                    if oss_name in self.direct_dep_list:
-                        comment = 'direct'
+                    dep_key = f"{oss_name}({oss_version})"
+                    if self.total_dep_list:
+                        if dep_key not in self.total_dep_list:
+                            continue
+                    if dep_key in self.direct_dep_list:
+                        comment_list.append('direct')
                     else:
-                        comment = 'transitive'
+                        comment_list.append('transitive')
+                    try:
+                        if dep_key in self.relation_tree:
+                            comment_list.extend(self.relation_tree[dep_key])
+                    except Exception as e:
+                        logger.error(f"Fail to find oss scope in dependency tree: {e}")
+                comment = ', '.join(comment_list)
 
                 sheet_list.append([manifest_file, oss_name, oss_version, license_name, dn_loc, homepage, '', '', comment])
 

src/fosslight_dependency/package_manager/Gradle.py

@@ -51,8 +51,9 @@ def parse_oss_information(self, f_name):
                 oss_name, oss_ini_version = parse_oss_name_version_in_filename(filename)
                 used_filename = True
 
+            dep_key = f"{oss_name}({oss_ini_version})"
             if self.total_dep_list:
-                if oss_name not in self.total_dep_list:
+                if dep_key not in self.total_dep_list:
                     continue
 
             oss_version = version_refine(oss_ini_version)
@@ -73,11 +74,19 @@ def parse_oss_information(self, f_name):
                 dn_loc = f"{self.dn_url}{group_id}/{artifact_id}/{oss_ini_version}"
                 homepage = f"{self.dn_url}{group_id}/{artifact_id}"
 
+            comment_list = []
             if self.direct_dep:
-                if oss_name in self.direct_dep_list:
-                    comment = 'direct'
-                else:
-                    comment = 'transitive'
+                if len(self.direct_dep_list) > 0:
+                    if dep_key in self.direct_dep_list:
+                        comment_list.append('direct')
+                    else:
+                        comment_list.append('transitive')
+                try:
+                    if dep_key in self.relation_tree:
+                        comment_list.extend(self.relation_tree[dep_key])
+                except Exception as e:
+                    logger.error(f"Fail to find oss scope in dependency tree: {e}")
+            comment = ', '.join(comment_list)
 
             sheet_list.append([const.SUPPORT_PACKAE.get(self.package_manager_name),
                               oss_name, oss_version, license_name, dn_loc, homepage, '', '', comment])

src/fosslight_dependency/package_manager/Maven.py

@@ -10,7 +10,6 @@
 from bs4 import BeautifulSoup as bs
 from xml.etree.ElementTree import parse
 import re
-import copy
 import fosslight_util.constant as constant
 import fosslight_dependency.constant as const
 from fosslight_dependency._package_manager import PackageManager
@@ -26,12 +25,10 @@ class Maven(PackageManager):
     input_file_name = os.path.join('target', 'generated-resources', 'licenses.xml')
     is_run_plugin = False
     output_custom_dir = ''
-    dependency_tree = {}
 
     def __init__(self, input_dir, output_dir, output_custom_dir):
         super().__init__(self.package_manager_name, self.dn_url, input_dir, output_dir)
         self.is_run_plugin = False
-        self.dependency_tree = {}
 
         if output_custom_dir:
             self.output_custom_dir = output_custom_dir
@@ -154,26 +151,60 @@ def run_maven_plugin(self):
         ret = subprocess.call(cmd, shell=True)
         if ret != 0:
             logger.error(f"Failed to run: {cmd}")
-            self.set_direct_dependencies(True)
+            self.set_direct_dependencies(False)
         else:
             self.parse_dependency_tree(dependency_tree_fname)
             self.set_direct_dependencies(True)
             os.remove(dependency_tree_fname)
 
+    def create_dep_stack(self, dep_line):
+        dep_stack = []
+        cur_flag = ''
+        dep_level = -1
+        dep_level_plus = False
+        for line in dep_line.readlines():
+            try:
+                if not line.startswith('[INFO]'):
+                    continue
+                if len(line) <= 7:
+                    continue
+                line = line[7:]
+
+                prev_flag = cur_flag
+                prev_dep_level = dep_level
+                dep_level = line.count("|")
+
+                re_result = re.findall(r'([\+|\\]\-)\s([^\:\s]+\:[^\:\s]+)\:(?:[^\:\s]+)\:([^\:\s]+)\:([^\:\s]+)', line)
+                if re_result:
+                    cur_flag = re_result[0][0]
+                    if (prev_flag == '\\-') and (prev_dep_level == dep_level):
+                        dep_level_plus = True
+                    if dep_level_plus and (prev_flag == '\\-') and (prev_dep_level != dep_level):
+                        dep_level_plus = False
+                    if dep_level_plus:
+                        dep_level += 1
+                    if re_result[0][3] == 'test':
+                        continue
+                    dep_name = f'{re_result[0][1]}({re_result[0][2]})'
+                    dep_stack = dep_stack[:dep_level] + [dep_name]
+                    yield dep_stack[:dep_level], dep_name
+                else:
+                    cur_flag = ''
+            except Exception as e:
+                logger.warning(f"Failed to parse dependency tree: {e}")
+
     def parse_dependency_tree(self, f_name):
         with open(f_name, 'r', encoding='utf8') as input_fp:
-            for i, line in enumerate(input_fp.readlines()):
-                try:
-                    line_bk = copy.deepcopy(line)
-                    re_result = re.findall(r'[\+|\\]\-\s([^\:\s]+\:[^\:\s]+)\:(?:[^\:\s]+)\:([^\:\s]+)\:([^\:\s]+)', line)
-                    if re_result:
-                        dependency_key = re_result[0][0] + ':' + re_result[0][1]
-                        if self.direct_dep:
-                            if re.match(r'^\[\w+\]\s[\+\\]\-', line_bk):
-                                self.direct_dep_list.append(re_result[0][0])
-                        self.dependency_tree[dependency_key] = re_result[0][2]
-                except Exception as e:
-                    logger.error(f"Failed to parse dependency tree: {e}")
+            try:
+                for stack, name in self.create_dep_stack(input_fp):
+                    if len(stack) == 0:
+                        self.direct_dep_list.append(name)
+                    else:
+                        if stack[-1] not in self.relation_tree:
+                            self.relation_tree[stack[-1]] = []
+                        self.relation_tree[stack[-1]].append(name)
+            except Exception as e:
+                logger.warning(f'Fail to parse maven dependency tree:{e}')
 
     def parse_oss_information(self, f_name):
         with open(f_name, 'r', encoding='utf8') as input_fp:
@@ -183,6 +214,7 @@ def parse_oss_information(self, f_name):
         dependencies = root.find("dependencies")
 
         sheet_list = []
+        comment = ''
 
         for d in dependencies.iter("dependency"):
             groupid = d.findtext("groupId")
@@ -205,20 +237,18 @@ def parse_oss_information(self, f_name):
                 # Case that doesn't include License tag value.
                 license_name = ''
 
+            dep_key = f"{oss_name}({version})"
             comment_list = []
-            try:
-                dependency_tree_key = f"{oss_name}:{version}"
-                if dependency_tree_key in self.dependency_tree.keys():
-                    comment_list.append(self.dependency_tree[dependency_tree_key])
-            except Exception as e:
-                logger.error(f"Fail to find oss scope in dependency tree: {e}")
-
             if self.direct_dep:
-                if oss_name in self.direct_dep_list:
+                if dep_key in self.direct_dep_list:
                     comment_list.append('direct')
                 else:
                     comment_list.append('transitive')
-
+                try:
+                    if dep_key in self.relation_tree:
+                        comment_list.extend(self.relation_tree[dep_key])
+                except Exception as e:
+                    logger.error(f"Fail to find oss scope in dependency tree: {e}")
             comment = ', '.join(comment_list)
 
             sheet_list.append([const.SUPPORT_PACKAE.get(self.package_manager_name),