Merge pull request #118 from fosslight/ossitem

Refactoring OSS item

Author: dd-jy

requirements.txt

@@ -4,8 +4,8 @@ openpyxl
 progress
 pyyaml
 beautifulsoup4
-fosslight_util~=1.4.48
-fosslight_source~=1.7.8
-fosslight_dependency~=3.15.1
-fosslight_binary~=4.1.30
+fosslight_util>=2.0.0
+fosslight_source>=2.0.0
+fosslight_dependency>=4.0.0
+fosslight_binary>=5.0.0
 fosslight_prechecker==3.0.27

src/fosslight_scanner/_run_compare.py

@@ -14,7 +14,8 @@
 from bs4 import BeautifulSoup
 import fosslight_util.constant as constant
 from fosslight_util.compare_yaml import compare_yaml
-from fosslight_util.convert_excel_to_yaml import convert_excel_to_yaml
+from fosslight_util.read_excel import read_oss_report
+from fosslight_util.parsing_yaml import parsing_yml
 
 logger = logging.getLogger(constant.LOGGER_NAME)
 ADD = "add"
@@ -255,10 +256,18 @@ def run_compare(before_f, after_f, output_path, output_file, file_ext, _start_ti
 
     result_file = get_comparison_result_filename(output_path, output_file, file_ext, _start_time)
 
-    if before_ext == XLSX_EXT:
-        convert_excel_to_yaml(before_f, before_yaml)
-        convert_excel_to_yaml(after_f, after_yaml)
-    compared_result = compare_yaml(before_yaml, after_yaml)
+    before_basepath = os.path.dirname(before_f)
+    after_basepath = os.path.dirname(after_f)
+    if XLSX_EXT == before_ext:
+        before_fileitems = read_oss_report(before_f, "", before_basepath)
+    elif YAML_EXT == before_ext:
+        before_fileitems, _, _ = parsing_yml(before_yaml, before_basepath)
+    if XLSX_EXT == after_ext:
+        after_fileitems = read_oss_report(after_f, after_basepath)
+    elif YAML_EXT == after_ext:
+        after_fileitems, _, _ = parsing_yml(after_yaml, after_basepath)
+
+    compared_result = compare_yaml(before_fileitems, after_fileitems)
     if compared_result != '':
         count_compared_result(compared_result)
         ret, result_file = write_compared_result(result_file, compared_result, file_ext, before_yaml, after_yaml)

src/fosslight_scanner/common.py

@@ -7,18 +7,12 @@
 import sys
 import logging
 import shutil
-import pandas as pd
-import yaml
-
-import fosslight_util.constant as constant
-from fosslight_util.parsing_yaml import parsing_yml
-from fosslight_util.write_yaml import create_yaml_with_ossitem
+import copy
+from fosslight_util.constant import LOGGER_NAME, FOSSLIGHT_SOURCE, FOSSLIGHT_BINARY, FOSSLIGHT_DEPENDENCY
 from fosslight_util.write_scancodejson import write_scancodejson
-from fosslight_util.read_excel import read_oss_report
-from fosslight_util.output_format import write_output_file
-from fosslight_util.oss_item import OssItem
+from fosslight_util.oss_item import OssItem, FileItem
 
-logger = logging.getLogger(constant.LOGGER_NAME)
+logger = logging.getLogger(LOGGER_NAME)
 SRC_SHEET = 'SRC_FL_Source'
 BIN_SHEET = 'BIN_FL_Binary'
 BIN_EXT_HEADER = {
@@ -93,72 +87,23 @@ def call_analysis_api(path_to_run, str_run_start, return_idx, func, *args, **kwa
     return success, result or []
 
 
-def overwrite_excel(excel_file_path, oss_name, column_name='OSS Name'):
-    if oss_name:
-        try:
-            files = os.listdir(excel_file_path)
-            for file in files:
-                if file.endswith(".xlsx"):
-                    file_path = os.path.join(excel_file_path, file)
-                    excel_file = pd.ExcelFile(file_path, engine='openpyxl')
-
-                    for sheet_name in excel_file.sheet_names:
-                        try:
-                            df = pd.read_excel(file_path, sheet_name=sheet_name, engine='openpyxl')
-                            if column_name in df.columns:
-                                updated = (df[column_name] == '') | (df[column_name].isnull())
-                                df.loc[updated, column_name] = oss_name
-                                df.to_excel(file_path, sheet_name=sheet_name, index=False)
-                        except Exception as ex:
-                            logger.debug(f"overwrite_sheet {sheet_name}:{ex}")
-        except Exception as ex:
-            logger.debug(f"overwrite_excel:{ex}")
-
-
-def merge_yamls(_output_dir, merge_yaml_files, final_report, remove_src_data=False,
-                default_oss_name='', default_oss_version='', url=''):
-    success = True
-    err_msg = ''
-
-    oss_total_list = []
-    yaml_dict = {}
-    try:
-        for mf in merge_yaml_files:
-            if os.path.exists(os.path.join(_output_dir, mf)):
-                oss_list, _, _ = parsing_yml(os.path.join(_output_dir, mf), _output_dir)
-
-                if remove_src_data:
-                    existed_yaml = {}
-                    for oi in oss_list:
-                        oi.name = default_oss_name if oi.name == '' else oi.name
-                        oi.version = default_oss_version if oi.version == '' else oi.version
-                        oi.download_location = url if oi.download_location == '' else oi.download_location
-                        create_yaml_with_ossitem(oi, existed_yaml)
-                    with open(os.path.join(_output_dir, mf), 'w') as f:
-                        yaml.dump(existed_yaml, f, default_flow_style=False, sort_keys=False)
-
-                oss_total_list.extend(oss_list)
-
-        if oss_total_list != []:
-            for oti in oss_total_list:
-                create_yaml_with_ossitem(oti, yaml_dict)
-            with open(os.path.join(_output_dir, final_report), 'w') as f:
-                yaml.dump(yaml_dict, f, default_flow_style=False, sort_keys=False)
-        else:
-            success = False
-            err_msg = "Output file is not created as no oss items detected."
-    except Exception as ex:
-        err_msg = ex
-        success = False
-
-    return success, err_msg
+def update_oss_item(scan_item, oss_name, oss_version, download_loc):
+    for file_items in scan_item.file_items.values():
+        for file_item in file_items:
+            if file_item.oss_items:
+                for oi in file_item.oss_items:
+                    if oi.name == '' and oi.version == '' and oi.download_location == '':
+                        oi.name = oss_name
+                        oi.version = oss_version
+                        oi.download_location = download_loc
+            else:
+                file_item.oss_items.append(OssItem(oss_name, oss_version, '', download_loc))
+    return scan_item
 
 
-def create_scancodejson(final_report, output_extension, ui_mode_report, src_path=""):
+def create_scancodejson(all_scan_item_origin, ui_mode_report, src_path=""):
     success = True
     err_msg = ''
-
-    oss_total_list = []
     root_dir = ""
     root_strip = ""
     try:
@@ -170,129 +115,96 @@ def create_scancodejson(final_report, output_extension, ui_mode_report, src_path
         root_dir = ""
 
     try:
-        item_without_oss = OssItem("")
-        oss_total_list = get_osslist(os.path.dirname(final_report), os.path.basename(final_report),
-                                     output_extension, '')
+        all_scan_item = copy.deepcopy(all_scan_item_origin)
+        if FOSSLIGHT_DEPENDENCY in all_scan_item.file_items:
+            del all_scan_item.file_items[FOSSLIGHT_DEPENDENCY]
         if src_path:
-            for root, dirs, files in os.walk(src_path):
+            fileitems_without_oss = []
+            for root, _, files in os.walk(src_path):
                 root = root.replace(root_strip, "")
                 for file in files:
+                    fi_without_oss = FileItem('')
+                    included = False
                     item_path = os.path.join(root, file)
                     item_path = item_path.replace(parent + os.path.sep, '', 1)
-                    included = any(item_path in x.source_name_or_path for x in oss_total_list)
+
+                    for file_items in all_scan_item.file_items.values():
+                        for file_item in file_items:
+                            if file_item.source_name_or_path:
+                                if file_item.source_name_or_path == item_path:
+                                    included = True
+                                    break
                     if not included:
-                        item_without_oss.source_name_or_path = item_path
-            if len(item_without_oss.source_name_or_path) > 0:
-                oss_total_list.append(item_without_oss)
+                        fi_without_oss.source_name_or_path = item_path
+                        fileitems_without_oss.append(fi_without_oss)
+            if len(fileitems_without_oss) > 0:
+                all_scan_item.file_items[FOSSLIGHT_SOURCE].extend(fileitems_without_oss)
         if root_dir:
-            for oss in oss_total_list:
-                tmp_path_list = oss.source_name_or_path
-                oss.source_name_or_path = ""
-                oss.source_name_or_path = [os.path.join(root_dir, path) for path in tmp_path_list]
-
+            for file_items in all_scan_item.file_items.values():
+                for fi in file_items:
+                    if fi.source_name_or_path:
+                        fi.source_name_or_path = os.path.join(root_dir, fi.source_name_or_path)
         write_scancodejson(os.path.dirname(ui_mode_report), os.path.basename(ui_mode_report),
-                           oss_total_list)
+                           all_scan_item)
     except Exception as ex:
         err_msg = ex
         success = False
 
     return success, err_msg
 
 
-def correct_scanner_result(_output_dir, output_files, output_extension, exist_src, exist_bin):
-    src_oss_list = []
-    bin_oss_list = []
+def correct_scanner_result(all_scan_item):
     duplicates = False
 
-    if exist_src:
-        src_oss_list = check_exclude_dir(get_osslist(_output_dir, output_files['SRC'], output_extension, SRC_SHEET))
-    if exist_bin:
-        bin_oss_list = check_exclude_dir(get_osslist(_output_dir, output_files['BIN'], output_extension, BIN_SHEET))
-
-    if exist_src and exist_bin:
+    keys_needed = {FOSSLIGHT_SOURCE, FOSSLIGHT_BINARY}
+    is_contained = keys_needed.issubset(all_scan_item.file_items.keys())
+    if is_contained:
+        src_fileitems = all_scan_item.file_items[FOSSLIGHT_SOURCE]
+        bin_fileitems = all_scan_item.file_items[FOSSLIGHT_BINARY]
         try:
             remove_src_idx_list = []
-            for idx_src, src_item in enumerate(src_oss_list):
+            for idx_src, src_fileitem in enumerate(src_fileitems):
+                src_fileitem.exclude = check_exclude_dir(src_fileitem.source_name_or_path, src_fileitem.exclude)
                 dup_flag = False
-                for bin_item in bin_oss_list:
-                    if (not src_item.source_name_or_path):
-                        continue
-                    if src_item.source_name_or_path[0] == bin_item.source_name_or_path[0]:
+                for bin_fileitem in bin_fileitems:
+                    bin_fileitem.exclude = check_exclude_dir(bin_fileitem.source_name_or_path, bin_fileitem.exclude)
+                    if src_fileitem.source_name_or_path == bin_fileitem.source_name_or_path:
                         dup_flag = True
-                        if not bin_item.license and src_item.license:
-                            src_item.exclude = bin_item.exclude
-                            bin_item.set_sheet_item(src_item.get_print_array(constant.FL_BINARY)[0])
-                            if bin_item.comment:
-                                bin_item.comment += '/'
-                            bin_item.comment += 'Loaded from SRC OSS info'
+                        src_all_licenses_non_empty = all(oss_item.license for oss_item in src_fileitem.oss_items)
+                        bin_empty_license_exists = all(not oss_item.license for oss_item in bin_fileitem.oss_items)
+
+                        if src_all_licenses_non_empty and bin_empty_license_exists:
+                            exclude = bin_fileitem.oss_items[0].exclude
+                            bin_fileitem.oss_items = []
+                            for src_oss_item in src_fileitem.oss_items:
+                                src_oss_item.exclude = exclude
+                                bin_fileitem.oss_items.append(src_oss_item)
+                            bin_fileitem.comment = 'Loaded from SRC OSS info'
                 if dup_flag:
                     remove_src_idx_list.append(idx_src)
             if remove_src_idx_list:
                 duplicates = True
                 for i in sorted(remove_src_idx_list, reverse=True):
-                    del src_oss_list[i]
+                    del src_fileitems[i]
         except Exception as ex:
             logger.warning(f"correct the scanner result:{ex}")
 
     try:
-        if exist_src:
-            success, err_msg = write_output_with_osslist(src_oss_list, _output_dir, output_files['SRC'],
-                                                         output_extension, SRC_SHEET)
-            if not success:
-                logger.warning(err_msg)
-        if exist_bin:
-            success, err_msg = write_output_with_osslist(bin_oss_list, _output_dir, output_files['BIN'],
-                                                         output_extension, BIN_SHEET, BIN_EXT_HEADER, BIN_HIDDEN_HEADER)
-            if not success:
-                logger.warning(err_msg)
         if duplicates:
             logger.info('Success to correct the src/bin scanner result')
     except Exception as ex:
         logger.warning(f"Corrected src/bin scanner result:{ex}")
-    return
-
-
-def write_output_with_osslist(oss_list, output_dir, output_file, output_extension, sheetname, extended_hdr={}, hidden_hdr={}):
-    new_oss_list = []
-    sheet_list = {}
-    sheet_list[sheetname] = []
-
-    for src_item in oss_list:
-        scanner_name = constant.supported_sheet_and_scanner[sheetname]
-        new_oss_list.append(src_item.get_print_array(scanner_name)[0])
-    sheet_list[sheetname].extend(new_oss_list)
-    if os.path.exists(os.path.join(output_dir, output_file)):
-        os.remove(os.path.join(output_dir, output_file))
-    success, err_msg, _ = write_output_file(os.path.join(output_dir, output_file).rstrip(output_extension),
-                                            output_extension, sheet_list, extended_hdr, hidden_hdr)
-    return success, err_msg
-
-
-def get_osslist(_output_dir, output_file, output_extension, sheet_name=''):
-    err_reason = ''
-    oss_list = []
-    oss_file_with_fullpath = os.path.join(_output_dir, output_file)
-
-    if os.path.exists(oss_file_with_fullpath):
-        if output_extension == '.xlsx':
-            oss_list = read_oss_report(oss_file_with_fullpath, sheet_name)
-        elif output_extension == '.yaml':
-            oss_list, _, err_reason = parsing_yml(oss_file_with_fullpath, _output_dir)
-        else:
-            err_reason = f'Not supported extension: {output_extension}'
-    if err_reason:
-        logger.info(f'get_osslist: {err_reason}')
-    return oss_list
+    return all_scan_item
 
 
-def check_exclude_dir(oss_list):
+def check_exclude_dir(source_name_or_path, file_item_exclude):
+    if file_item_exclude:
+        return True
     _exclude_dirs = ["venv", "node_modules", "Pods", "Carthage"]
+    exclude = False
 
-    for oss_item in oss_list:
-        if not oss_item.source_name_or_path:
-            continue
-        for exclude_dir in _exclude_dirs:
-            if exclude_dir in oss_item.source_name_or_path[0].split(os.path.sep):
-                oss_item.exclude = True
-                break
-    return oss_list
+    for exclude_dir in _exclude_dirs:
+        if exclude_dir in source_name_or_path.split(os.path.sep):
+            exclude = True
+            break
+    return exclude