Add dependency recursive mode

Signed-off-by: 석지영/책임연구원/SW공학(연)Open Source TP <[email protected]>

Author: dd-jy

src/fosslight_scanner/_help.py

@@ -40,13 +40,14 @@
             --no_correction\t    Enter if you don't want to correct OSS information with sbom-info.yaml
                                      * Correction mode only supported xlsx format.
             --correct_fpath <path>  Path to the sbom-info.yaml file
-            --ui\t\t\t  Generate UI mode result file
+            --ui\t\t    Generate UI mode result file
+            --recursive_dep\t    Recursively analyze dependencies
 
         Options for only 'all' or 'bin' mode
             -u <db_url>\t\t    DB Connection(format :'postgresql://username:password@host:port/database_name')
 
         Options for only 'all' or 'dependency' mode
-            -d <dependency_argument>\t    Additional arguments for running dependency analysis"""
+            -d <dependency_arg>\t    Additional arguments for running dependency analysis"""
 
 
 def print_help_msg():

src/fosslight_scanner/_parse_setting.py

@@ -25,12 +25,13 @@ def parse_setting_json(data):
     source_print_matched_text = data.get('source_print_matched_text', False)
     source_time_out = data.get('source_time_out', 120)
     binary_simple = data.get('binary_simple', False)
+    recursive_dep = data.get('recursive_dep', False)
     str_lists = [mode, path, exclude_path]
     strings = [
         dep_argument, output, format, db_url,
         correct_fpath, link, selected_source_scanner
     ]
-    booleans = [timer, raw, no_correction, ui, source_write_json_file, source_print_matched_text, binary_simple]
+    booleans = [timer, raw, no_correction, ui, source_write_json_file, source_print_matched_text, binary_simple, recursive_dep]
 
     is_incorrect = False
 
@@ -65,4 +66,4 @@ def parse_setting_json(data):
     return mode, path, dep_argument, output, format, link, db_url, timer, \
         raw, core, no_correction, correct_fpath, ui, exclude_path, \
         selected_source_scanner, source_write_json_file, source_print_matched_text, source_time_out, \
-        binary_simple
+        binary_simple, recursive_dep

src/fosslight_scanner/cli.py

@@ -15,7 +15,8 @@
 
 
 def set_args(mode, path, dep_argument, output, format, link, db_url, timer,
-             raw, core, no_correction, correct_fpath, ui, setting, exclude_path):
+             raw, core, no_correction, correct_fpath, ui, setting, exclude_path,
+             recursive_dep):
 
     selected_source_scanner = "all"
     source_write_json_file = False
@@ -30,7 +31,7 @@ def set_args(mode, path, dep_argument, output, format, link, db_url, timer,
             s_mode, s_path, s_dep_argument, s_output, s_format, s_link, s_db_url, s_timer, s_raw, s_core, \
                 s_no_correction, s_correct_fpath, s_ui, s_exclude_path, \
                 s_selected_source_scanner, s_source_write_json_file, s_source_print_matched_text, \
-                s_source_time_out, s_binary_simple = parse_setting_json(data)
+                s_source_time_out, s_binary_simple, s_recursive_dep = parse_setting_json(data)
 
             # direct cli arguments have higher priority than setting file
             mode = mode or s_mode
@@ -47,6 +48,7 @@ def set_args(mode, path, dep_argument, output, format, link, db_url, timer,
             correct_fpath = correct_fpath or s_correct_fpath
             ui = ui or s_ui
             exclude_path = exclude_path or s_exclude_path
+            recursive_dep = recursive_dep or s_recursive_dep
 
             # These options are only set from the setting file, not from CLI arguments
             selected_source_scanner = s_selected_source_scanner or selected_source_scanner
@@ -60,7 +62,7 @@ def set_args(mode, path, dep_argument, output, format, link, db_url, timer,
     return mode, path, dep_argument, output, format, link, db_url, timer, \
         raw, core, no_correction, correct_fpath, ui, exclude_path, \
         selected_source_scanner, source_write_json_file, source_print_matched_text, source_time_out, \
-        binary_simple
+        binary_simple, recursive_dep
 
 
 def main():
@@ -79,7 +81,7 @@ def main():
                         type=str, dest='format',nargs='*', default=[])
     parser.add_argument('--output', '-o', help='Output directory or file',
                         type=str, dest='output', default="")
-    parser.add_argument('--dependency', '-d', help='Dependency arguments',
+    parser.add_argument('--dependency', '-d', help='Dependency arguments (e.g. -d "-m pip" )',
                         type=str, dest='dep_argument', default="")
     parser.add_argument('--url', '-u', help="DB Url",
                         type=str, dest='db_url', default="")
@@ -105,6 +107,8 @@ def main():
                         type=str, required=False, default='')
     parser.add_argument('--ui', help='Generate UI mode result file',
                         action='store_true', required=False, default=False)
+    parser.add_argument('--recursive_dep', '-rd', help='Recursively analyze dependencies',
+                        action='store_true', dest='recursive_dep', default=False)
 
     try:
         args = parser.parse_args()
@@ -118,16 +122,16 @@ def main():
     else:
         mode, path, dep_argument, output, format, link, db_url, timer, raw, core, no_correction, correct_fpath, \
             ui, exclude_path, selected_source_scanner, source_write_json_file, source_print_matched_text, \
-            source_time_out, binary_simple, = set_args(
+            source_time_out, binary_simple, recursive_dep = set_args(
                 args.mode, args.path, args.dep_argument, args.output,
                 args.format, args.link, args.db_url, args.timer, args.raw,
                 args.core, args.no_correction, args.correct_fpath, args.ui,
-                args.setting, args.exclude_path)
+                args.setting, args.exclude_path, args.recursive_dep)
 
         run_main(mode, path, dep_argument, output, format, link, db_url, timer,
                  raw, core, not no_correction, correct_fpath, ui, exclude_path,
                  selected_source_scanner, source_write_json_file, source_print_matched_text,
-                 source_time_out, binary_simple)
+                 source_time_out, binary_simple, recursive_dep)
 
 
 if __name__ == "__main__":

src/fosslight_scanner/fosslight_scanner.py

@@ -57,7 +57,8 @@
 ]
 
 
-def run_dependency(path_to_analyze, output_file_with_path, params="", path_to_exclude=[], formats=[]):
+def run_dependency(path_to_analyze, output_file_with_path, params="", path_to_exclude=[], formats=[],
+                   recursive_dep=False):
     result = []
 
     package_manager = ""
@@ -100,7 +101,9 @@ def run_dependency(path_to_analyze, output_file_with_path, params="", path_to_ex
             output_file_with_path,
             pip_activate_cmd, pip_deactivate_cmd,
             output_custom_dir, app_name,
-            github_token, formats, True, path_to_exclude=path_to_exclude
+            github_token, formats, True, path_to_exclude=path_to_exclude,
+            graph_path="", graph_size=(600,600),
+            recursive=recursive_dep
         )
         if success:
             result = scan_item
@@ -131,7 +134,7 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
                 default_oss_name="", default_oss_version="", url="",
                 correct_mode=True, correct_fpath="", ui_mode=False, path_to_exclude=[],
                 selected_source_scanner="all", source_write_json_file=False, source_print_matched_text=False,
-                source_time_out=120, binary_simple=False, formats=[]):
+                source_time_out=120, binary_simple=False, formats=[], recursive_dep=False):
     final_excel_dir = output_path
     success = True
     all_cover_items = []
@@ -232,7 +235,8 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
 
             if run_dep:
                 dep_scanitem = run_dependency(src_path, _output_dir,
-                                              dep_arguments, path_to_exclude, formats)
+                                              dep_arguments, path_to_exclude, formats,
+                                              recursive_dep)
                 all_scan_item.file_items.update(dep_scanitem.file_items)
                 all_cover_items.append(dep_scanitem.cover)
         else:
@@ -359,7 +363,7 @@ def run_main(mode_list, path_arg, dep_arguments, output_file_or_dir, file_format
              db_url, hide_progressbar=False, keep_raw_data=False, num_cores=-1,
              correct_mode=True, correct_fpath="", ui_mode=False, path_to_exclude=[],
              selected_source_scanner="all", source_write_json_file=False, source_print_matched_text=False,
-             source_time_out=120, binary_simple=False):
+             source_time_out=120, binary_simple=False, recursive_dep=False):
     global _executed_path, _start_time
 
     output_files = []
@@ -470,7 +474,7 @@ def run_main(mode_list, path_arg, dep_arguments, output_file_or_dir, file_format
                                 default_oss_name, default_oss_version, url_to_analyze,
                                 correct_mode, correct_fpath, ui_mode, path_to_exclude,
                                 selected_source_scanner, source_write_json_file, source_print_matched_text, source_time_out,
-                                binary_simple, formats)
+                                binary_simple, formats, recursive_dep)
 
                 if extract_folder:
                     shutil.rmtree(extract_folder)

tests/test__parse_setting.py

@@ -31,5 +31,5 @@ def test_parse_setting_json_valid_data():
     result = parse_setting_json(data)
     assert result == (
         ['test'], ['/some/path'], 'arg', 'output', 'json', 'http://example.com', 'sqlite:///:memory:', True,
-        True, 4, True, '/correct/path', True, ['/exclude/path'], 'scanner', True, True, 60, True
+        True, 4, True, '/correct/path', True, ['/exclude/path'], 'scanner', True, True, 60, True, False
     )

tests/test_cli.py

@@ -45,13 +45,14 @@ def mock_open(*args, **kwargs):
     # Call the function with some arguments
     result = set_args(
         mode=None, path=None, dep_argument=None, output=None, format=None, link=None, db_url=None, timer=None,
-        raw=None, core=-1, no_correction=None, correct_fpath=None, ui=None, setting="dummy_path", exclude_path=None
+        raw=None, core=-1, no_correction=None, correct_fpath=None, ui=None, setting="dummy_path", exclude_path=None,
+        recursive_dep=False
     )
 
     # Expected result
     expected = (
         ["test_mode"], ["test_path"], "test_dep_argument", "test_output", ["test_format"], "test_link", "test_db_url", True,
-        True, 4, True, "test_correct_fpath", True, ["test_exclude_path"], "test_scanner", True, True, 100, True
+        True, 4, True, "test_correct_fpath", True, ["test_exclude_path"], "test_scanner", True, True, 100, True, False
     )
 
     assert result == expected