Merge pull request #38 from fosslight/develop

Add compare mode

Author: dd-jy

.reuse/dep5

@@ -25,3 +25,7 @@ License: Apache-2.0
 Files: tests/*
 Copyright: 2021 LG Electronics
 License: Apache-2.0
+
+Files: src/fosslight_scanner/resources/bom_compare.html
+Copyright: 2022 LG Electronics
+License: Apache-2.0

requirements.txt

@@ -5,6 +5,7 @@ xlrd
 openpyxl
 progress
 pyyaml
+beautifulsoup4
 fosslight_util>=1.3.13
 fosslight_dependency>=3.7.4
 fosslight_binary>=4.0.7

setup.py

@@ -32,6 +32,7 @@
                      "Programming Language :: Python :: 3.8",
                      "Programming Language :: Python :: 3.9", ],
         install_requires=required,
+        package_data={'fosslight_scanner': ['resources/bom_compare.html']},
         entry_points={
             "console_scripts": [
                 "fosslight = fosslight_scanner.cli:main",

src/fosslight_scanner/_help.py

@@ -18,12 +18,14 @@
             binary\t\t    Run FOSSLight Binary
             reuse\t\t    Run FOSSLight Reuse
             all\t\t\t    Run all scanners
+            compare\t\t    Compare two FOSSLight reports in yaml format
 
         Options:
             -h\t\t\t    Print help message
             -p <path>\t\t    Path to analyze
             -w <link>\t\t    Link to be analyzed can be downloaded by wget or git clone
-            -f <format>\t\t    Output file format (excel, csv, opossum)
+            -f <format>\t\t    FOSSLight Report file format (excel, yaml)
+                               \t\t(In compare mode, supports excel, json, yaml, html)
             -o <output>\t\t    Output directory or file
             -c <number>\t\t    Number of processes to analyze source
             -r\t\t\t    Keep raw data
@@ -34,7 +36,10 @@
             -u <db_url>\t\t    DB Connection(format :'postgresql://username:password@host:port/database_name')
 
         Options for only 'all' or 'dependency' mode
-            -d <dependency_argument>\t    Additional arguments for running dependency analysis"""
+            -d <dependency_argument>\t    Additional arguments for running dependency analysis
+
+        Options for only 'compare' mode
+            -y <before yaml> <after yaml>   Two FOSSLight reports in yaml format (ex, -y 'before.yaml' 'after.yaml')"""
 
 
 def print_help_msg():

src/fosslight_scanner/_run_compare.py

@@ -0,0 +1,189 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# FOSSLight Scanner
+# Copyright (c) 2022 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+import os
+import sys
+import json
+import yaml
+import logging
+import xlsxwriter
+import codecs
+from pathlib import Path
+from bs4 import BeautifulSoup
+import fosslight_util.constant as constant
+from fosslight_util.compare_yaml import compare_yaml
+
+logger = logging.getLogger(constant.LOGGER_NAME)
+ADD = "add"
+DELETE = "delete"
+CHANGE = "change"
+
+
+def write_result_json_yaml(output_file, compared_result, file_ext):
+    ret = True
+    try:
+        with open(output_file, 'w') as f:
+            if file_ext == '.json':
+                json.dump(compared_result, f, indent=4, sort_keys=True)
+            elif file_ext == '.yaml':
+                yaml.dump(compared_result, f, sort_keys=True)
+    except Exception:
+        ret = False
+    return ret
+
+
+def parse_result_for_table(oi, status):
+    compared_row = []
+    if status == ADD:
+        oss_after = f"{oi['name']} ({oi['version']})"
+        license_after = f"{', '.join(oi['license'])}"
+        compared_row = [status, '', '', oss_after, license_after]
+    elif status == DELETE:
+        oss_before = f"{oi['name']} ({oi['version']})"
+        license_before = f"{', '.join(oi['license'])}"
+        compared_row = [status, oss_before, license_before, '', '']
+    elif status == CHANGE:
+        oss_before, oss_after, license_before, license_after = [], [], [], []
+        for prev_i in oi['prev']:
+            oss_before.append(f"{oi['name']} ({prev_i['version']})")
+            license_before.append(f"{', '.join(prev_i['license'])}")
+        for now_i in oi['now']:
+            oss_after.append(f"{oi['name']} ({now_i['version']})")
+            license_after.append(f"{', '.join(now_i['license'])}")
+        compared_row = [status, ' / '.join(oss_before), ' / '.join(license_before),
+                                ' / '.join(oss_after), ' / '.join(license_after)]
+    else:
+        logger.error(f"Not supported compared status: {status}")
+
+    return compared_row
+
+
+def get_sample_html():
+    RESOURCES_DIR = 'resources'
+    SAMPLE_HTML = 'bom_compare.html'
+    html_file = os.path.join(RESOURCES_DIR, SAMPLE_HTML)
+    html_f = ''
+
+    try:
+        base_dir = sys._MEIPASS
+    except Exception:
+        base_dir = os.path.dirname(__file__)
+
+    file_withpath = os.path.join(base_dir, html_file)
+    try:
+        html_f = codecs.open(file_withpath, 'r', 'utf-8')
+    except Exception as ex:
+        logger.error(f"Error to get sample html file : {ex}")
+
+    return html_f
+
+
+def write_result_html(output_file, compared_result, before_yaml, after_yaml):
+    ret = True
+    html_f = get_sample_html()
+    if html_f != '':
+        try:
+            f = BeautifulSoup(html_f.read(), 'html.parser')
+            f.find("li", {"class": "before_f"}).append(before_yaml)
+            f.find("li", {"class": "after_f"}).append(after_yaml)
+
+            table_html = f.find("table", {"id": "comp_result"})
+
+            status = [ADD, DELETE, CHANGE]
+            row = 2
+            for st in status:
+                for oi in compared_result[st]:
+                    compared_row = parse_result_for_table(oi, st)
+                    tr = f.new_tag('tr')
+                    for i, ci in enumerate(compared_row):
+                        td = f.new_tag('td')
+                        td.string = ci
+                        td.attrs = {"style": "padding:5px;"}
+                        tr.insert(i, td)
+                    table_html.insert(row, tr)
+                    row += 1
+
+            if row == 2:
+                tr = f.new_tag('tr')
+                td = f.new_tag('td')
+                td.string = 'Same'
+                td.attrs = {"style": "padding:5px;"}
+                tr.insert(0, td)
+                for i in range(1, 5):
+                    td = f.new_tag('td')
+                    td.string = ''
+                    td.attrs = {"style": "padding:5px;"}
+                    tr.insert(i, td)
+                table_html.insert(row, tr)
+
+            with open(output_file, "wb") as f_out:
+                f_out.write(f.prettify("utf-8"))
+        except Exception as e:
+            logger.error(f'Fail to write html file: {e}')
+            ret = False
+    else:
+        ret = False
+    return ret
+
+
+def write_result_xlsx(output_file, compared_result):
+    HEADER = ['Status', 'OSS_Before', 'License_Before', 'OSS_After', 'License_After']
+    ret = True
+
+    try:
+        output_dir = os.path.dirname(output_file)
+        Path(output_dir).mkdir(parents=True, exist_ok=True)
+
+        workbook = xlsxwriter.Workbook(os.path.basename(output_file))
+        worksheet = workbook.add_worksheet('BOM_compare')
+        bold = workbook.add_format({'bold': True})
+        worksheet.write_row(0, 0, HEADER, bold)
+
+        row = 1
+        status = [ADD, DELETE, CHANGE]
+        for st in status:
+            for oi in compared_result[st]:
+                compared_row = parse_result_for_table(oi, st)
+                worksheet.write_row(row, 0, compared_row)
+                row += 1
+        if row == 1:
+            worksheet.write_row(row, 0, ['Same', '', '', '', ''])
+        workbook.close()
+    except Exception as e:
+        logger.error(f'Fail to write xlsx file: {e}')
+        ret = False
+
+    return ret
+
+
+def write_compared_result(output_file, compared_result, file_ext, before_yaml='', after_yaml=''):
+    success = False
+    if file_ext == "" or file_ext == ".xlsx":
+        success = write_result_xlsx(output_file, compared_result)
+    elif file_ext == ".html":
+        success = write_result_html(output_file, compared_result, before_yaml, after_yaml)
+    elif file_ext == ".json":
+        success = write_result_json_yaml(output_file, compared_result, file_ext)
+    elif file_ext == ".yaml":
+        success = write_result_json_yaml(output_file, compared_result, file_ext)
+    else:
+        logger.info("Not supported file extension")
+
+    return success
+
+
+def run_compare(before_yaml, after_yaml, output_file, file_ext):
+    ret = False
+    logger.info("Start compare mode")
+
+    compared_result = compare_yaml(before_yaml, after_yaml)
+    if compared_result != '':
+        ret = write_compared_result(output_file, compared_result, file_ext, before_yaml, after_yaml)
+        if ret:
+            logger.info(f"Success to write compared result: {output_file}")
+        else:
+            logger.error("Fail to write compared result file.")
+
+    return ret

src/fosslight_scanner/cli.py

@@ -10,10 +10,10 @@
 
 def main():
     parser = ArgumentParser(description='FOSSLight Scanner', prog='fosslight_scanner', add_help=False)
-    parser.add_argument('mode', nargs='?', help='source| dependency| binary| reuse| all', default="all")
+    parser.add_argument('mode', nargs='?', help='source| dependency| binary| reuse| all| compare', default="all")
     parser.add_argument('--path', '-p', help='Path to analyze', type=str, dest='path', default="")
     parser.add_argument('--wget', '-w', help='Link to be analyzed', type=str, dest='link', default="")
-    parser.add_argument('--file', '-f', help='Output file format (excel, csv, opossum)', type=str, dest='file', default="")
+    parser.add_argument('--file', '-f', help='Output file format (excel, csv, opossum, yaml)', type=str, dest='file', default="")
     parser.add_argument('--output', '-o', help='Output directory or file', type=str, dest='output', default="")
     parser.add_argument('--dependency', '-d', help='Dependency arguments', type=str, dest='dep_argument', default="")
     parser.add_argument('--url', '-u', help="DB Url", type=str, dest='db_url', default="")
@@ -22,6 +22,7 @@ def main():
     parser.add_argument('--timer', '-t', help='Hide the progress bar',  action='store_true', dest='timer', default=False)
     parser.add_argument('--version', '-v', help='Print version',  action='store_true', dest='version', default=False)
     parser.add_argument('--help', '-h', help='Print help message', action='store_true', dest='help')
+    parser.add_argument('--yaml', '-y', help='Two FOSSLight reports in yaml format', nargs=2, default="")
     try:
         args = parser.parse_args()
     except SystemExit:
@@ -32,8 +33,15 @@ def main():
     elif args.version:
         print_package_version(PKG_NAME, "FOSSLight Scanner Version:")
     else:
+        if args.yaml:
+            before_yaml = args.yaml[0]
+            after_yaml = args.yaml[1]
+        else:
+            before_yaml = ''
+            after_yaml = ''
+
         run_main(args.mode, args.path, args.dep_argument, args.output, args.file,
-                 args.link, args.db_url, args.timer, args.raw, args.core)
+                 args.link, args.db_url, args.timer, args.raw, args.core, before_yaml, after_yaml)
 
 
 if __name__ == "__main__":