Support recursive dependency analysis (#264)

Signed-off-by: 석지영/책임연구원/SW공학(연)Open Source TP <[email protected]>

Author: dd-jy

src/fosslight_dependency/_analyze_dependency.py

@@ -92,11 +92,11 @@ def analyze_dependency(package_manager_name, input_dir, output_dir, pip_activate
             package_manager.parse_oss_information_for_pnpm()
             package_dep_item_list.extend(package_manager.dep_items)
     if ret:
-        logger.warning(f"### Complete to analyze: {package_manager_name}")
+        logger.warning(f"### Complete to analyze: {package_manager_name}({input_dir}: {','.join(manifest_file_name)})")
         if package_manager.cover_comment:
             cover_comment = package_manager.cover_comment
     else:
-        logger.error(f"### Fail to analyze: {package_manager_name}")
+        logger.error(f"### Fail to analyze: {package_manager_name}({input_dir}: {','.join(manifest_file_name)})")
 
     del package_manager
 

src/fosslight_dependency/_help.py

@@ -46,6 +46,7 @@
             \t\t\t\t\t--graph-path option is required
             --direct\t\t\t    Print the direct/transitive dependency type in comment.
                                 \t\tChoice 'True' or 'False'. (default:True)
+            -r\t\t\t\t    Recursive mode. Scan all subdirectories for manifest files.
             --notice\t\t\t    Print the open source license notice text.
 
         Required only for swift, carthage

src/fosslight_dependency/_package_manager.py

@@ -108,6 +108,7 @@ def run_gradle_task(self):
                     cmd_gradle = "./gradlew"
             else:
                 ret_task = False
+                self.set_direct_dependencies(False)
                 logger.warning('No gradlew file exists (Skip to find dependencies relationship.).')
                 if ret_plugin:
                     logger.warning('Also it cannot run android-dependency-scanning plugin.')

src/fosslight_dependency/package_manager/Npm.py

@@ -31,12 +31,12 @@ def __init__(self, input_dir, output_dir):
         super().__init__(self.package_manager_name, self.dn_url, input_dir, output_dir)
 
     def __del__(self):
-        if os.path.isfile(self.input_file_name):
-            os.remove(self.input_file_name)
+        if os.path.isfile(os.path.join(self.input_dir, self.input_file_name)):
+            os.remove(os.path.join(self.input_dir, self.input_file_name))
         if self.flag_tmp_node_modules:
-            shutil.rmtree(node_modules, ignore_errors=True)
-        if os.path.exists(self.tmp_custom_json):
-            os.remove(self.tmp_custom_json)
+            shutil.rmtree(os.path.join(self.input_dir, node_modules), ignore_errors=True)
+        if os.path.exists(os.path.join(self.input_dir, self.tmp_custom_json)):
+            os.remove(os.path.join(self.input_dir, self.tmp_custom_json))
 
     def run_plugin(self):
         ret = self.start_license_checker()
@@ -122,9 +122,9 @@ def parse_transitive_relationship(self):
                 if len(rel_json) < 1:
                     ret = False
                 else:
-                    self.package_name = f'{rel_json[_name]}({rel_json[_version]})'
+                    self.package_name = f'{rel_json[_name]}({rel_json.get(_version, "")})'
                     if _dependencies in rel_json:
-                        self.parse_rel_dependencies(rel_json[_name], rel_json[_version], rel_json[_dependencies])
+                        self.parse_rel_dependencies(rel_json[_name], rel_json.get(_version, ""), rel_json[_dependencies])
             except Exception as e:
                 ret = False
                 err_msg = e

src/fosslight_dependency/run_dependency_scanner.py

@@ -30,7 +30,7 @@
                                        'OSS Version', 'License', 'Download Location',
                                        'Homepage', 'Copyright Text', 'Exclude',
                                        'Comment', 'Depends On']}
-_exclude_dir = ['node_moduels', 'venv']
+_exclude_dir = ['node_modules', 'venv', 'Pods', 'Carthage']
 
 
 def get_terminal_size():
@@ -50,7 +50,7 @@ def paginate_file(file_path):
             input("Press Enter to see the next page...")
 
 
-def find_package_manager(input_dir, abs_path_to_exclude=[], manifest_file_name=[]):
+def find_package_manager(input_dir, abs_path_to_exclude=[], manifest_file_name=[], recursive=False):
     ret = True
     if not manifest_file_name:
         for value in const.SUPPORT_PACKAE.values():
@@ -62,9 +62,8 @@ def find_package_manager(input_dir, abs_path_to_exclude=[], manifest_file_name=[
     found_manifest_file = []
     suggested_files = []
     for parent, dirs, files in os.walk(input_dir):
-        if len(files) < 1:
-            continue
-        if os.path.basename(parent) in _exclude_dir:
+        parent_parts = parent.split(os.sep)
+        if any(ex_dir in parent_parts for ex_dir in _exclude_dir):
             continue
         if os.path.abspath(parent) in abs_path_to_exclude:
             continue
@@ -75,7 +74,8 @@ def find_package_manager(input_dir, abs_path_to_exclude=[], manifest_file_name=[
                    for exclude_path in abs_path_to_exclude):
                 continue
             if file in manifest_file_name:
-                found_manifest_file.append(file)
+                path_with_filename = os.path.join(parent, file)
+                found_manifest_file.append(path_with_filename)
             if file in const.SUGGESTED_PACKAGE.keys():
                 suggested_files.append(os.path.join(parent, file))
         for dir in dirs:
@@ -84,42 +84,53 @@ def find_package_manager(input_dir, abs_path_to_exclude=[], manifest_file_name=[
                 if len(manifest_l) > 1:
                     if manifest_l[0] == dir:
                         if os.path.exists(os.path.join(parent, manifest_f)):
-                            found_manifest_file.append(manifest_f)
-        if len(found_manifest_file) > 0:
-            input_dir = parent
-            break
-    found_package_manager = defaultdict(list)
-    for f_idx in found_manifest_file:
+                            found_manifest_file.append(os.path.join(parent, manifest_f))
+        if not recursive:
+            if len(found_manifest_file) > 0:
+                input_dir = parent
+                break
+
+    found_package_manager = defaultdict(lambda: defaultdict(list))
+    for f_with_path in found_manifest_file:
+        f_name = os.path.basename(f_with_path)
+        dir_path = os.path.dirname(f_with_path)
         for key, value in const.SUPPORT_PACKAE.items():
             if isinstance(value, list):
-                for v in value:
-                    if f_idx == v:
-                        if key in found_package_manager.keys():
-                            found_package_manager[key].append(f_idx)
-                        else:
-                            found_package_manager[key] = [f_idx]
+                if f_name in value:
+                    found_package_manager[key][dir_path].append(f_name)
             else:
-                if value == f_idx:
-                    found_package_manager[key] = [f_idx]
+                if f_name == value:
+                    found_package_manager[key][dir_path].append(f_name)
+    found_package_manager = {k: dict(v) for k, v in found_package_manager.items()}
 
     # both npm and pnpm are detected, remove npm.
-    if 'npm' in found_package_manager and 'pnpm' in found_package_manager:
+    if 'npm' in found_package_manager.keys() and 'pnpm' in found_package_manager.keys():
         del found_package_manager['npm']
     if len(found_package_manager) >= 1:
-        manifest_file_w_path = [os.path.join(input_dir, file) for pkg, files in found_package_manager.items() for file in files]
-        logger.info(f"Found the manifest file({','.join(manifest_file_w_path)}) automatically.")
-        logger.warning(f"### Set Package Manager = {', '.join(found_package_manager.keys())}")
+        log_lines = ["\nDetected Manifest Files automatically"]
+        log_lines = print_package_info(found_package_manager, log_lines)
+        logger.info('\n'.join(log_lines))
     else:
         ret = False
         logger.info("Cannot find the manifest file.")
 
     return ret, found_package_manager, input_dir, suggested_files
 
 
+def print_package_info(success_pm, log_lines):
+    if success_pm:
+        for pm, dir_dict in success_pm.items():
+            log_lines.append(f"- {pm}:")
+            for path, files in dir_dict.items():
+                file_list = ', '.join(files)
+                log_lines.append(f"  {path}: {file_list}")
+    return log_lines
+
+
 def run_dependency_scanner(package_manager='', input_dir='', output_dir_file='', pip_activate_cmd='',
                            pip_deactivate_cmd='', output_custom_dir='', app_name=const.default_app_name,
                            github_token='', formats=[], direct=True, path_to_exclude=[], graph_path='',
-                           graph_size=(600, 600)):
+                           graph_size=(600, 600), recursive=False):
     global logger
 
     ret = True
@@ -217,9 +228,8 @@ def run_dependency_scanner(package_manager='', input_dir='', output_dir_file='',
     try:
         ret, found_package_manager, input_dir, suggested_files = find_package_manager(input_dir,
                                                                                       abs_path_to_exclude,
-                                                                                      manifest_file_name)
-        if ret:
-            os.chdir(input_dir)
+                                                                                      manifest_file_name,
+                                                                                      recursive)
     except Exception as e:
         if autodetect:
             logger.error(f'Fail to find package manager: {e}')
@@ -228,7 +238,7 @@ def run_dependency_scanner(package_manager='', input_dir='', output_dir_file='',
         if not ret:
             if not autodetect:
                 logger.info('Try to analyze dependency without manifest file. (Manual mode)')
-                found_package_manager[package_manager] = []
+                found_package_manager[package_manager] = {}
             else:
                 ret = False
                 if suggested_files:
@@ -243,40 +253,53 @@ def run_dependency_scanner(package_manager='', input_dir='', output_dir_file='',
                 else:
                     scan_item.set_cover_comment("No Package manager detected.")
 
-    pass_key = 'PASS'
-    success_pm = []
-    fail_pm = []
+    pass_key = ['PASS']
+    success_pm = defaultdict(lambda: defaultdict(list))
+    fail_pm = defaultdict(lambda: defaultdict(list))
     cover_comment = ''
-    for pm, manifest_file_name in found_package_manager.items():
-        if manifest_file_name == pass_key:
-            continue
-        ret, package_dep_item_list, cover_comment = analyze_dependency(pm, input_dir, output_path,
-                                                                       pip_activate_cmd, pip_deactivate_cmd,
-                                                                       output_custom_dir, app_name, github_token,
-                                                                       manifest_file_name, direct)
-        if ret:
-            success_pm.append(f"{pm} ({', '.join(manifest_file_name)})")
-            scan_item.append_file_items(package_dep_item_list)
-            if pm == const.GRADLE:
-                if const.ANDROID in found_package_manager.keys():
-                    found_package_manager[const.ANDROID] = pass_key
-                    if f"{const.ANDROID} ({', '.join(manifest_file_name)})" in fail_pm:
-                        fail_pm.remove(f"{const.ANDROID} ({', '.join(manifest_file_name)})")
-            elif pm == const.ANDROID:
-                if const.GRADLE in found_package_manager.keys():
-                    found_package_manager[const.GRADLE] = pass_key
-                    if f"{const.GRADLE} ({', '.join(manifest_file_name)})" in fail_pm:
-                        fail_pm.remove(f"{const.GRADLE} ({', '.join(manifest_file_name)})")
-        else:
-            fail_pm.append(f"{pm} ({', '.join(manifest_file_name)})")
+    for pm, manifest_file_name_list in found_package_manager.items():
+        for manifest_dir, manifest_file_name in manifest_file_name_list.items():
+            input_dir = manifest_dir
+            if manifest_file_name == pass_key:
+                continue
+            os.chdir(input_dir)
+            ret, package_dep_item_list, cover_comment = analyze_dependency(pm, input_dir, output_path,
+                                                                           pip_activate_cmd, pip_deactivate_cmd,
+                                                                           output_custom_dir, app_name, github_token,
+                                                                           manifest_file_name, direct)
+            if ret:
+                success_pm[pm][input_dir].extend(manifest_file_name)
+                scan_item.append_file_items(package_dep_item_list)
+
+                dup_pm = None
+                if pm == const.GRADLE and const.ANDROID in found_package_manager:
+                    dup_pm = const.ANDROID
+                elif pm == const.ANDROID and const.GRADLE in found_package_manager:
+                    dup_pm = const.GRADLE
+
+                if dup_pm:
+                    if dup_pm in fail_pm and input_dir in fail_pm[dup_pm]:
+                        fail_pm[dup_pm].pop(input_dir, None)
+                        if not fail_pm[dup_pm]:
+                            fail_pm.pop(dup_pm, None)
+                    else:
+                        found_package_manager[dup_pm][manifest_dir] = pass_key
+            else:
+                fail_pm[pm][input_dir].extend(manifest_file_name)
 
+    success_pm = {k: dict(v) for k, v in success_pm.items()}
+    fail_pm = {k: dict(v) for k, v in fail_pm.items()}
     if len(found_package_manager.keys()) > 0:
         if len(success_pm) > 0:
-            scan_item.set_cover_comment(f"Analyzed Package manager: {', '.join(success_pm)}")
+            log_lines = ["Success to analyze:"]
+            log_lines = print_package_info(success_pm, log_lines)
+            scan_item.set_cover_comment('\n'.join(log_lines))
         if len(fail_pm) > 0:
-            info_msg = 'Check log file(fosslight_log*.txt) ' \
-                       'and https://fosslight.org/fosslight-guide-en/scanner/3_dependency.html#-prerequisite.'
-            scan_item.set_cover_comment(f"Analysis failed Package manager: {', '.join(fail_pm)} ({info_msg})")
+            log_lines = ["Fail to analyze:"]
+            log_lines = print_package_info(fail_pm, log_lines)
+            scan_item.set_cover_comment('\n'.join(log_lines))
+            scan_item.set_cover_comment('Check log file(fosslight_log*.txt) '
+                                        'and https://fosslight.org/fosslight-guide-en/scanner/3_dependency.html#-prerequisite.')
 
     if ret and graph_path:
         graph_path = os.path.abspath(graph_path)
@@ -335,6 +358,7 @@ def main():
     graph_path = ''
     graph_size = (600, 600)
     direct = True
+    recursive = False
 
     parser = argparse.ArgumentParser(add_help=False)
     parser.add_argument('-h', '--help', action='store_true', required=False)
@@ -353,6 +377,7 @@ def main():
     parser.add_argument('--graph-size', nargs=2, type=int, metavar=("WIDTH", "HEIGHT"), required=False)
     parser.add_argument('--direct', choices=('true', 'false'), default='True', required=False)
     parser.add_argument('--notice', action='store_true', required=False)
+    parser.add_argument('-r', '--recursive', action='store_true', required=False)
 
     args = parser.parse_args()
 
@@ -405,10 +430,12 @@ def main():
             paginate_file(source_file)
             shutil.copyfile(source_file, destination_file)
         sys.exit(0)
+    if args.recursive:  # -r option
+        recursive = True
 
     run_dependency_scanner(package_manager, input_dir, output_dir, pip_activate_cmd, pip_deactivate_cmd,
                            output_custom_dir, app_name, github_token, format, direct, path_to_exclude,
-                           graph_path, graph_size)
+                           graph_path, graph_size, recursive)
 
 
 if __name__ == '__main__':