Merge pull request #42 from fosslight/develop

Support yaml format of FOSSLight Report

Author: dd-jy

src/fosslight_scanner/_run_compare.py

@@ -19,6 +19,7 @@
 ADD = "add"
 DELETE = "delete"
 CHANGE = "change"
+COMP_STATUS = [ADD, DELETE, CHANGE]
 
 
 def write_result_json_yaml(output_file, compared_result, file_ext):
@@ -93,10 +94,9 @@ def write_result_html(output_file, compared_result, before_yaml, after_yaml):
 
             table_html = f.find("table", {"id": "comp_result"})
 
-            status = [ADD, DELETE, CHANGE]
             row = 2
             MIN_ROW_NUM = 100
-            for st in status:
+            for st in COMP_STATUS:
                 for oi in compared_result[st]:
                     compared_row = parse_result_for_table(oi, st)
                     tr = f.new_tag('tr')
@@ -155,8 +155,7 @@ def write_result_xlsx(output_file, compared_result):
         worksheet.write_row(0, 0, HEADER, bold)
 
         row = 1
-        status = [ADD, DELETE, CHANGE]
-        for st in status:
+        for st in COMP_STATUS:
             for oi in compared_result[st]:
                 compared_row = parse_result_for_table(oi, st)
                 worksheet.write_row(row, 0, compared_row)
@@ -215,6 +214,16 @@ def get_comparison_result_filename(output_path, output_file, output_extension, _
     return result_file
 
 
+def count_compared_result(compared_result):
+    comp_len = [len(compared_result[st]) for st in COMP_STATUS]
+    if sum(comp_len) == 0:
+        count_str = "all oss lists are the same."
+    else:
+        count_str = f"total {sum(comp_len)} oss updated ("
+        count_str += ', '.join([f"{COMP_STATUS[x]}: {comp_len[x]}" for x in range(0, 3)]) + ")"
+    logger.info(f"Comparison result: {count_str}")
+
+
 def run_compare(before_yaml, after_yaml, output_path, output_file, file_ext, _start_time):
     ret = False
     logger.info("Start compare mode")
@@ -224,6 +233,7 @@ def run_compare(before_yaml, after_yaml, output_path, output_file, file_ext, _st
     result_file = get_comparison_result_filename(output_path, output_file, file_ext, _start_time)
     compared_result = compare_yaml(before_yaml, after_yaml)
     if compared_result != '':
+        count_compared_result(compared_result)
         ret, result_file = write_compared_result(result_file, compared_result, file_ext, before_yaml, after_yaml)
         if ret:
             logger.info(f"Success to write compared result: {result_file}")

src/fosslight_scanner/common.py

@@ -9,7 +9,10 @@
 from shutil import copy
 import re
 import pandas as pd
+import yaml
 import fosslight_util.constant as constant
+from fosslight_util.parsing_yaml import parsing_yml
+from fosslight_util.write_yaml import create_yaml_with_ossitem
 
 logger = logging.getLogger(constant.LOGGER_NAME)
 
@@ -154,3 +157,40 @@ def overwrite_excel(excel_file_path, oss_name, column_name='OSS Name'):
                             logger.debug(f"overwrite_sheet {sheet_name}:{ex}")
         except Exception as ex:
             logger.debug(f"overwrite_excel:{ex}")
+
+
+def merge_yamls(_output_dir, merge_yaml_files, final_report, remove_src_data=False, default_oss_name='', url=''):
+    success = True
+    err_msg = ''
+
+    oss_total_list = []
+    yaml_dict = {}
+    try:
+        for mf in merge_yaml_files:
+            if os.path.exists(os.path.join(_output_dir, mf)):
+                oss_list, license_list = parsing_yml(os.path.join(_output_dir, mf), _output_dir)
+
+                if remove_src_data:
+                    existed_yaml = {}
+                    for oi in oss_list:
+                        oi.name = default_oss_name if oi.name == '-' else oi.name
+                        oi.download_location = url if oi.download_location == '' else oi.download_location
+                        create_yaml_with_ossitem(oi, existed_yaml)
+                    with open(os.path.join(_output_dir, mf), 'w') as f:
+                        yaml.dump(existed_yaml, f, default_flow_style=False, sort_keys=False)
+
+                oss_total_list.extend(oss_list)
+
+        if oss_total_list != []:
+            for oti in oss_total_list:
+                create_yaml_with_ossitem(oti, yaml_dict)
+            with open(os.path.join(_output_dir, final_report), 'w') as f:
+                yaml.dump(yaml_dict, f, default_flow_style=False, sort_keys=False)
+        else:
+            success = False
+            err_msg = "Output file is not created as no oss items detected."
+    except Exception as ex:
+        err_msg = ex
+        success = False
+
+    return success, err_msg

src/fosslight_scanner/fosslight_scanner.py

@@ -22,7 +22,8 @@
 from fosslight_util.output_format import check_output_format
 from fosslight_prechecker._precheck import run_lint as prechecker_lint
 from .common import (copy_file, call_analysis_api,
-                     overwrite_excel, extract_name_from_link)
+                     overwrite_excel, extract_name_from_link,
+                     merge_yamls)
 from fosslight_util.write_excel import merge_excels
 from ._run_compare import run_compare
 import subprocess
@@ -125,8 +126,8 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
         abs_path = os.path.abspath(src_path)
 
         if success:
-            output_files = {"SRC": "FL_Source.xlsx",
-                            "BIN": "FL_Binary.xlsx",
+            output_files = {"SRC": f"FL_Source{output_extension}",
+                            "BIN": f"FL_Binary{output_extension}",
                             "BIN_TXT": "FL_Binary.txt",
                             "DEP": f"FL_Dependency{output_extension}",
                             "PRECHECKER": "FL_Prechecker.yaml"}
@@ -151,7 +152,9 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
                                                             False, num_cores, True)
                         if success:
                             sheet_list["SRC_FL_Source"] = [scan_item.get_row_to_print() for scan_item in result[2]]
-                            create_report_file(0, result[2], result[3], 'all', True, _output_dir, output_files["SRC"], "")
+                            need_license = True if output_extension == ".xlsx" else False
+                            create_report_file(0, result[2], result[3], 'all', need_license,
+                                               _output_dir, output_files["SRC"].split('.')[0], output_extension)
                     else:  # Run fosslight_source by using docker image
                         src_output = os.path.join("output", output_files["SRC"])
                         output_rel_path = os.path.relpath(abs_path, os.getcwd())
@@ -186,10 +189,15 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
     try:
         output_file_without_ext = os.path.join(final_excel_dir, output_file)
         final_report = f"{output_file_without_ext}{output_extension}"
-        if remove_src_data:
-            overwrite_excel(_output_dir, default_oss_name, "OSS Name")
-            overwrite_excel(_output_dir, url, "Download Location")
-        success, err_msg = merge_excels(_output_dir, final_report)
+        if output_extension == ".xlsx":
+            if remove_src_data:
+                overwrite_excel(_output_dir, default_oss_name, "OSS Name")
+                overwrite_excel(_output_dir, url, "Download Location")
+            success, err_msg = merge_excels(_output_dir, final_report)
+        elif output_extension == ".yaml":
+            merge_yaml_files = [output_files["SRC"], output_files["BIN"], output_files["DEP"]]
+            success, err_msg = merge_yamls(_output_dir, merge_yaml_files, final_report,
+                                           remove_src_data, default_oss_name, url)
 
         if success:
             result_log["Output File"] = final_report