Update scanner with ossinfo

Signed-off-by: jiyeong.seok <[email protected]>

Author: dd-jy

src/fosslight_scanner/common.py

@@ -9,16 +9,13 @@
 import shutil
 import pandas as pd
 import yaml
-
-import fosslight_util.constant as constant
+from fosslight_util.constant import LOGGER_NAME, FOSSLIGHT_SOURCE, FOSSLIGHT_BINARY
 from fosslight_util.parsing_yaml import parsing_yml
-from fosslight_util.write_yaml import create_yaml_with_ossitem
 from fosslight_util.write_scancodejson import write_scancodejson
 from fosslight_util.read_excel import read_oss_report
-from fosslight_util.output_format import write_output_file
 from fosslight_util.oss_item import OssItem
 
-logger = logging.getLogger(constant.LOGGER_NAME)
+logger = logging.getLogger(LOGGER_NAME)
 SRC_SHEET = 'SRC_FL_Source'
 BIN_SHEET = 'BIN_FL_Binary'
 BIN_EXT_HEADER = {
@@ -93,65 +90,18 @@ def call_analysis_api(path_to_run, str_run_start, return_idx, func, *args, **kwa
     return success, result or []
 
 
-def overwrite_excel(excel_file_path, oss_name, column_name='OSS Name'):
-    if oss_name:
-        try:
-            files = os.listdir(excel_file_path)
-            for file in files:
-                if file.endswith(".xlsx"):
-                    file_path = os.path.join(excel_file_path, file)
-                    excel_file = pd.ExcelFile(file_path, engine='openpyxl')
-
-                    for sheet_name in excel_file.sheet_names:
-                        try:
-                            df = pd.read_excel(file_path, sheet_name=sheet_name, engine='openpyxl')
-                            if column_name in df.columns:
-                                updated = (df[column_name] == '') | (df[column_name].isnull())
-                                df.loc[updated, column_name] = oss_name
-                                df.to_excel(file_path, sheet_name=sheet_name, index=False)
-                        except Exception as ex:
-                            logger.debug(f"overwrite_sheet {sheet_name}:{ex}")
-        except Exception as ex:
-            logger.debug(f"overwrite_excel:{ex}")
-
-
-def merge_yamls(_output_dir, merge_yaml_files, final_report, remove_src_data=False,
-                default_oss_name='', default_oss_version='', url=''):
-    success = True
-    err_msg = ''
-
-    oss_total_list = []
-    yaml_dict = {}
-    try:
-        for mf in merge_yaml_files:
-            if os.path.exists(os.path.join(_output_dir, mf)):
-                oss_list, _, _ = parsing_yml(os.path.join(_output_dir, mf), _output_dir)
-
-                if remove_src_data:
-                    existed_yaml = {}
-                    for oi in oss_list:
-                        oi.name = default_oss_name if oi.name == '' else oi.name
-                        oi.version = default_oss_version if oi.version == '' else oi.version
-                        oi.download_location = url if oi.download_location == '' else oi.download_location
-                        create_yaml_with_ossitem(oi, existed_yaml)
-                    with open(os.path.join(_output_dir, mf), 'w') as f:
-                        yaml.dump(existed_yaml, f, default_flow_style=False, sort_keys=False)
-
-                oss_total_list.extend(oss_list)
-
-        if oss_total_list != []:
-            for oti in oss_total_list:
-                create_yaml_with_ossitem(oti, yaml_dict)
-            with open(os.path.join(_output_dir, final_report), 'w') as f:
-                yaml.dump(yaml_dict, f, default_flow_style=False, sort_keys=False)
-        else:
-            success = False
-            err_msg = "Output file is not created as no oss items detected."
-    except Exception as ex:
-        err_msg = ex
-        success = False
-
-    return success, err_msg
+def update_oss_item(scan_item, oss_name, oss_version, download_loc):
+    for file_items in scan_item.file_items.values():
+        for file_item in file_items:
+            if file_item.oss_items:
+                for oi in file_item.oss_items:
+                    if oi.name == '' and oi.version == '' and oi.download_location == '':
+                        oi.name = oss_name
+                        oi.version = oss_version
+                        oi.download_location = download_loc
+            else:
+                file_item.oss_items.append(OssItem(oss_name, oss_version, '', download_loc))
+    return scan_item
 
 
 def create_scancodejson(final_report, output_extension, ui_mode_report, src_path=""):
@@ -199,73 +149,48 @@ def create_scancodejson(final_report, output_extension, ui_mode_report, src_path
     return success, err_msg
 
 
-def correct_scanner_result(_output_dir, output_files, output_extension, exist_src, exist_bin):
-    src_oss_list = []
-    bin_oss_list = []
+def correct_scanner_result(all_scan_item):
     duplicates = False
 
-    if exist_src:
-        src_oss_list = check_exclude_dir(get_osslist(_output_dir, output_files['SRC'], output_extension, SRC_SHEET))
-    if exist_bin:
-        bin_oss_list = check_exclude_dir(get_osslist(_output_dir, output_files['BIN'], output_extension, BIN_SHEET))
-
-    if exist_src and exist_bin:
+    keys_needed = {FOSSLIGHT_SOURCE, FOSSLIGHT_BINARY}
+    is_contained = keys_needed.issubset(all_scan_item.file_items.keys())
+    if is_contained:
+        src_fileitems = all_scan_item.file_items[FOSSLIGHT_SOURCE]
+        bin_fileitems = all_scan_item.file_items[FOSSLIGHT_BINARY]
         try:
             remove_src_idx_list = []
-            for idx_src, src_item in enumerate(src_oss_list):
+            for idx_src, src_fileitem in enumerate(src_fileitems):
+                src_fileitem.exclude = check_exclude_dir(src_fileitem.source_name_or_path)
                 dup_flag = False
-                for bin_item in bin_oss_list:
-                    if (not src_item.source_name_or_path):
-                        continue
-                    if src_item.source_name_or_path[0] == bin_item.source_name_or_path[0]:
+                for bin_fileitem in bin_fileitems:
+                    bin_fileitem.exclude = check_exclude_dir(bin_fileitem.source_name_or_path)
+                    if src_fileitem.source_name_or_path == bin_fileitem.source_name_or_path:
                         dup_flag = True
-                        if not bin_item.license and src_item.license:
-                            src_item.exclude = bin_item.exclude
-                            bin_item.set_sheet_item(src_item.get_print_array(constant.FL_BINARY)[0])
-                            if bin_item.comment:
-                                bin_item.comment += '/'
-                            bin_item.comment += 'Loaded from SRC OSS info'
+                        src_all_licenses_non_empty = all(oss_item.license for oss_item in src_fileitem.oss_items)
+                        bin_empty_license_exists = all(not oss_item.license for oss_item in bin_fileitem.oss_items)
+
+                        if src_all_licenses_non_empty and bin_empty_license_exists:
+                            exclude = bin_fileitem.oss_items[0].exclude
+                            bin_fileitem.oss_items = []
+                            for src_oss_item in src_fileitem.oss_items:
+                                src_oss_item.exclude = exclude
+                                bin_fileitem.oss_items.append(src_oss_item)
+                            bin_fileitem.comment = 'Loaded from SRC OSS info'
                 if dup_flag:
                     remove_src_idx_list.append(idx_src)
             if remove_src_idx_list:
                 duplicates = True
                 for i in sorted(remove_src_idx_list, reverse=True):
-                    del src_oss_list[i]
+                    del src_fileitems[i]
         except Exception as ex:
             logger.warning(f"correct the scanner result:{ex}")
 
     try:
-        if exist_src:
-            success, err_msg = write_output_with_osslist(src_oss_list, _output_dir, output_files['SRC'],
-                                                         output_extension, SRC_SHEET)
-            if not success:
-                logger.warning(err_msg)
-        if exist_bin:
-            success, err_msg = write_output_with_osslist(bin_oss_list, _output_dir, output_files['BIN'],
-                                                         output_extension, BIN_SHEET, BIN_EXT_HEADER, BIN_HIDDEN_HEADER)
-            if not success:
-                logger.warning(err_msg)
         if duplicates:
             logger.info('Success to correct the src/bin scanner result')
     except Exception as ex:
         logger.warning(f"Corrected src/bin scanner result:{ex}")
-    return
-
-
-def write_output_with_osslist(oss_list, output_dir, output_file, output_extension, sheetname, extended_hdr={}, hidden_hdr={}):
-    new_oss_list = []
-    sheet_list = {}
-    sheet_list[sheetname] = []
-
-    for src_item in oss_list:
-        scanner_name = constant.supported_sheet_and_scanner[sheetname]
-        new_oss_list.append(src_item.get_print_array(scanner_name)[0])
-    sheet_list[sheetname].extend(new_oss_list)
-    if os.path.exists(os.path.join(output_dir, output_file)):
-        os.remove(os.path.join(output_dir, output_file))
-    success, err_msg, _ = write_output_file(os.path.join(output_dir, output_file).rstrip(output_extension),
-                                            output_extension, sheet_list, extended_hdr, hidden_hdr)
-    return success, err_msg
+    return all_scan_item
 
 
 def get_osslist(_output_dir, output_file, output_extension, sheet_name=''):
@@ -285,14 +210,12 @@ def get_osslist(_output_dir, output_file, output_extension, sheet_name=''):
     return oss_list
 
 
-def check_exclude_dir(oss_list):
+def check_exclude_dir(source_name_or_path):
     _exclude_dirs = ["venv", "node_modules", "Pods", "Carthage"]
+    exclude = False
 
-    for oss_item in oss_list:
-        if not oss_item.source_name_or_path:
-            continue
-        for exclude_dir in _exclude_dirs:
-            if exclude_dir in oss_item.source_name_or_path[0].split(os.path.sep):
-                oss_item.exclude = True
-                break
-    return oss_list
+    for exclude_dir in _exclude_dirs:
+        if exclude_dir in source_name_or_path.split(os.path.sep):
+            exclude = True
+            break
+    return exclude

src/fosslight_scanner/fosslight_scanner.py

@@ -25,12 +25,13 @@
 import fosslight_util.constant as constant
 from fosslight_util.output_format import check_output_format
 from fosslight_prechecker._precheck import run_lint as prechecker_lint
-from fosslight_util.write_excel import merge_excels, merge_cover_comment
 from fosslight_util.cover import CoverItem
+from fosslight_util.oss_item import ScannerItem
+from fosslight_util.output_format import write_output_file
 
 from .common import (
-    copy_file, call_analysis_api, overwrite_excel,
-    merge_yamls, correct_scanner_result, create_scancodejson
+    copy_file, call_analysis_api, update_oss_item,
+    correct_scanner_result, create_scancodejson
 )
 from ._run_compare import run_compare
 
@@ -56,7 +57,7 @@
 
 
 def run_dependency(path_to_analyze, output_file_with_path, params="", path_to_exclude=[]):
-    result_list = []
+    result = []
 
     package_manager = ""
     pip_activate_cmd = ""
@@ -90,7 +91,7 @@ def run_dependency(path_to_analyze, output_file_with_path, params="", path_to_ex
     timer.start()
 
     try:
-        success, result = call_analysis_api(
+        success, scan_item = call_analysis_api(
             path_to_analyze, "Dependency Analysis",
             1, run_dependency_scanner,
             package_manager,
@@ -101,11 +102,11 @@ def run_dependency(path_to_analyze, output_file_with_path, params="", path_to_ex
             github_token, path_to_exclude=path_to_exclude
         )
         if success:
-            result_list = result.get('SRC_FL_Dependency')
+            result = scan_item
     except Exception as ex:
         logger.warning(f"Run dependency: {ex}")
 
-    return result_list or []
+    return result
 
 
 def source_analysis_wrapper(*args, **kwargs):
@@ -130,7 +131,8 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
                 source_time_out=120, binary_simple=False):
     final_excel_dir = output_path
     success = True
-    temp_output_fiiles = []
+    all_cover_items = []
+    all_scan_item = ScannerItem(PKG_NAME, _start_time)
     if not remove_src_data:
         success, final_excel_dir, result_log = init(output_path)
 
@@ -158,9 +160,6 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
                                                     -1, prechecker_lint,
                                                     abs_path, False, output_prechecker,
                                                     exclude_path=path_to_exclude)
-                success_file, copied_file = copy_file(output_prechecker, output_path)
-                if success_file:
-                    temp_output_fiiles.append(copied_file)
 
             if run_src:
                 try:
@@ -180,6 +179,9 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
                                     source_print_matched_text=source_print_matched_text,
                                     source_time_out=source_time_out
                         )
+                        if success:
+                            all_scan_item.file_items.update(result[2].file_items)
+                            all_cover_items.append(result[2].cover)
 
                     else:  # Run fosslight_source by using docker image
                         src_output = os.path.join("output", output_files["SRC"])
@@ -195,16 +197,22 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
                     logger.warning(f"Failed to run source analysis: {ex}")
 
             if run_bin:
-                success, _ = call_analysis_api(src_path, "Binary Analysis",
-                                               1, binary_analysis.find_binaries,
-                                               abs_path,
-                                               os.path.join(_output_dir, output_files["BIN"]),
-                                               "", db_url, binary_simple,
-                                               correct_mode, correct_fpath,
-                                               path_to_exclude=path_to_exclude)
+                success, result = call_analysis_api(src_path, "Binary Analysis",
+                                            1, binary_analysis.find_binaries,
+                                            abs_path,
+                                            os.path.join(_output_dir, output_files["BIN"]),
+                                            "", db_url, binary_simple,
+                                            correct_mode, correct_fpath,
+                                            path_to_exclude=path_to_exclude)
+                if success:
+                    all_scan_item.file_items.update(result.file_items)
+                    all_cover_items.append(result.cover)
 
             if run_dep:
-                run_dependency(src_path, os.path.join(_output_dir, output_files["DEP"]), dep_arguments, path_to_exclude)
+                dep_scanitem = run_dependency(src_path, os.path.join(_output_dir, output_files["DEP"]),
+                                              dep_arguments, path_to_exclude)
+                all_scan_item.file_items.update(dep_scanitem.file_items)
+                all_cover_items.append(dep_scanitem.cover)
 
         else:
             return
@@ -215,46 +223,23 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
     try:
         output_file_without_ext = os.path.join(final_excel_dir, output_file)
         final_report = f"{output_file_without_ext}{output_extension}"
-        merge_files = [output_files["SRC"], output_files["BIN"], output_files["DEP"]]
         cover = CoverItem(tool_name=PKG_NAME,
                           start_time=_start_time,
                           input_path=abs_path,
                           exclude_path=path_to_exclude,
                           simple_mode=False)
-        cover.comment = merge_cover_comment(_output_dir, merge_files)
-
-        if output_extension == ".xlsx":
-            tmp_dir = f"tmp_{datetime.now().strftime('%y%m%d_%H%M')}"
-            exist_src = False
-            exist_bin = False
-            if correct_mode:
-                os.makedirs(os.path.join(_output_dir, tmp_dir), exist_ok=True)
-                if os.path.exists(os.path.join(_output_dir, output_files['SRC'])):
-                    exist_src = True
-                    shutil.copy2(os.path.join(_output_dir, output_files['SRC']), os.path.join(_output_dir, tmp_dir))
-                if os.path.exists(os.path.join(_output_dir, output_files['BIN'])):
-                    exist_bin = True
-                    shutil.copy2(os.path.join(_output_dir, output_files['BIN']), os.path.join(_output_dir, tmp_dir))
-                if exist_src or exist_bin:
-                    correct_scanner_result(_output_dir, output_files, output_extension, exist_src, exist_bin)
-
-            if remove_src_data:
-                overwrite_excel(_output_dir, default_oss_name, "OSS Name")
-                overwrite_excel(_output_dir, default_oss_version, "OSS Version")
-                overwrite_excel(_output_dir, url, "Download Location")
-            success, err_msg = merge_excels(_output_dir, final_report, merge_files, cover)
-
-            if correct_mode:
-                if exist_src:
-                    shutil.move(os.path.join(_output_dir, tmp_dir, output_files['SRC']),
-                                os.path.join(_output_dir, output_files['SRC']))
-                if exist_bin:
-                    shutil.move(os.path.join(_output_dir, tmp_dir, output_files['BIN']),
-                                os.path.join(_output_dir, output_files['BIN']))
-                shutil.rmtree(os.path.join(_output_dir, tmp_dir), ignore_errors=True)
-        elif output_extension == ".yaml":
-            success, err_msg = merge_yamls(_output_dir, merge_files, final_report,
-                                           remove_src_data, default_oss_name, default_oss_version, url)
+        merge_comment = []
+        for ci in all_cover_items:
+            merge_comment.append(str(f'[{ci.tool_name}] {ci.comment}'))
+        cover.comment = '\n'.join(merge_comment)
+        all_scan_item.cover = cover
+
+        if correct_mode:
+            all_scan_item = correct_scanner_result(all_scan_item)
+
+        if remove_src_data:
+            all_scan_item = update_oss_item(all_scan_item, default_oss_name, default_oss_version, url)
+        success, err_msg, final_report = write_output_file(output_file_without_ext, output_extension, all_scan_item)
         if success:
             if os.path.isfile(final_report):
                 logger.info(f'Generated the result file: {final_report}')