Support spdx

requirements.txt

@@ -1,8 +1,8 @@
 pyparsing
-scancode-toolkit>=32.0.2,==32.0.*
+scancode-toolkit>=32.2.0
 scanoss
 XlsxWriter
-fosslight_util>=2.0.0
+fosslight_util>=2.1.0
 PyYAML
 wheel>=0.38.1
 intbitset

src/fosslight_source/_parsing_scanoss_file.py

@@ -45,6 +45,7 @@ def parsing_scanResult(scanoss_report: dict, path_to_scan: str = "", path_to_exc
         if any(os.path.commonpath([abs_file_path, exclude_path]) == exclude_path for exclude_path in abs_path_to_exclude):
             continue
         result_item = SourceItem(file_path)
+
         if 'id' in findings[0]:
             if "none" == findings[0]['id']:
                 continue

src/fosslight_source/_scan_item.py

@@ -7,7 +7,7 @@
 import logging
 import re
 import fosslight_util.constant as constant
-from fosslight_util.oss_item import FileItem, OssItem
+from fosslight_util.oss_item import FileItem, OssItem, get_checksum_sha1
 
 logger = logging.getLogger(constant.LOGGER_NAME)
 replace_word = ["-only", "-old-style", "-or-later", "licenseref-scancode-", "licenseref-"]
@@ -42,6 +42,8 @@ def __init__(self, value: str) -> None:
         self.oss_name = ""
         self.oss_version = ""
 
+        self.checksum = get_checksum_sha1(value)
+
     def __del__(self) -> None:
         pass
 

src/fosslight_source/cli.py

@@ -5,6 +5,7 @@
 
 import sys
 import os
+import platform
 import warnings
 import logging
 from datetime import datetime
@@ -13,7 +14,7 @@
 from fosslight_util.timer_thread import TimerThread
 from ._help import print_version, print_help_msg_source_scanner
 from ._license_matched import get_license_list_to_print
-from fosslight_util.output_format import check_output_formats, write_output_file
+from fosslight_util.output_format import check_output_formats_v2, write_output_file
 from fosslight_util.correct import correct_with_yaml
 from .run_scancode import run_scan
 from .run_scanoss import run_scanoss_py
@@ -49,7 +50,7 @@ def main() -> None:
     write_json_file = False
     output_file_name = ""
     print_matched_text = False
-    formats = ""
+    formats = []
     selected_scanner = ""
     correct_mode = True
 
@@ -145,7 +146,8 @@ def create_report_file(
     selected_scanner: str, need_license: bool = False,
     output_path: str = "", output_files: list = [],
     output_extensions: list = [], correct_mode: bool = True,
-    correct_filepath: str = "", path_to_scan: str = "", path_to_exclude: list = []
+    correct_filepath: str = "", path_to_scan: str = "", path_to_exclude: list = [],
+    formats: list = []
 ) -> 'ScannerItem':
     """
     Create report files for given scanned result.
@@ -168,12 +170,33 @@ def create_report_file(
         # If -o does not contains file name, set default name
         while len(output_files) < len(output_extensions):
             output_files.append(None)
+        to_remove = []  # elements of spdx format on windows that should be removed
         for i, output_extension in enumerate(output_extensions):
             if output_files[i] is None or output_files[i] == "":
-                if output_extension == _json_ext:
-                    output_files[i] = f"fosslight_opossum_src_{_start_time}"
+                if formats:
+                    if formats[i].startswith('spdx'):
+                        if platform.system() != 'Windows':
+                            output_files[i] = f"fosslight_spdx_src_{_start_time}"
+                        else:
+                            logger.warning('spdx format is not supported on Windows. Please remove spdx from format.')
+                            to_remove.append(i)
+                    else:
+                        if output_extension == _json_ext:
+                            output_files[i] = f"fosslight_opossum_src_{_start_time}"
+                        else:
+                            output_files[i] = f"fosslight_report_src_{_start_time}"
                 else:
-                    output_files[i] = f"fosslight_report_src_{_start_time}"
+                    if output_extension == _json_ext:
+                        output_files[i] = f"fosslight_opossum_src_{_start_time}"
+                    else:
+                        output_files[i] = f"fosslight_report_src_{_start_time}"
+        for index in sorted(to_remove, reverse=True):
+            # remove elements of spdx format on windows
+            del output_files[index]
+            del output_extensions[index]
+            del formats[index]
+        if len(output_extensions) < 1:
+            sys.exit(0)
 
     if not correct_filepath:
         correct_filepath = path_to_scan
@@ -219,10 +242,10 @@ def create_report_file(
 
     combined_paths_and_files = [os.path.join(output_path, file) for file in output_files]
     results = []
-    for combined_path_and_file, output_extension in zip(combined_paths_and_files, output_extensions):
+    for combined_path_and_file, output_extension, output_format in zip(combined_paths_and_files, output_extensions, formats):
         # if need_license and output_extension == _json_ext and "scanoss_reference" in sheet_list:
         #     del sheet_list["scanoss_reference"]
-        results.append(write_output_file(combined_path_and_file, output_extension, scan_item, extended_header, ""))
+        results.append(write_output_file(combined_path_and_file, output_extension, scan_item, extended_header, "", output_format))
     for success, msg, result_file in results:
         if success:
             logger.info(f"Output file: {result_file}")
@@ -295,8 +318,9 @@ def run_scanners(
     license_list = []
     spdx_downloads = {}
     result_log = {}
+    scan_item = []
 
-    success, msg, output_path, output_files, output_extensions = check_output_formats(output_file_name, formats)
+    success, msg, output_path, output_files, output_extensions, formats = check_output_formats_v2(output_file_name, formats)
 
     logger, result_log = init_log(os.path.join(output_path, f"fosslight_log_src_{start_time}.txt"),
                                   True, logging.INFO, logging.DEBUG, PKG_NAME, path_to_scan, path_to_exclude)
@@ -319,7 +343,7 @@ def run_scanners(
             merged_result = merge_results(scancode_result, scanoss_result, spdx_downloads)
             scan_item = create_report_file(start_time, merged_result, license_list, scanoss_result, selected_scanner,
                                            print_matched_text, output_path, output_files, output_extensions, correct_mode,
-                                           correct_filepath, path_to_scan, path_to_exclude)
+                                           correct_filepath, path_to_scan, path_to_exclude, formats)
         else:
             print_help_msg_source_scanner()
             result_log[RESULT_KEY] = "Unsupported scanner"

src/fosslight_source/run_scancode.py

@@ -14,7 +14,7 @@
 from fosslight_util.set_log import init_log
 from ._parsing_scancode_file_item import parsing_file_item
 from ._parsing_scancode_file_item import get_error_from_header
-from fosslight_util.output_format import check_output_formats
+from fosslight_util.output_format import check_output_formats_v2
 from fosslight_binary.binary_analysis import check_binary
 from typing import Tuple
 
@@ -46,7 +46,7 @@ def run_scan(
     if not correct_filepath:
         correct_filepath = path_to_scan
 
-    success, msg, output_path, output_files, output_extensions = check_output_formats(output_file_name, formats)
+    success, msg, output_path, output_files, output_extensions, formats = check_output_formats_v2(output_file_name, formats)
     if success:
         if output_path == "":  # if json output with _write_json_file not used, output_path won't be needed.
             output_path = os.getcwd()

src/fosslight_source/run_scanoss.py

@@ -11,7 +11,7 @@
 from datetime import datetime
 import fosslight_util.constant as constant
 from fosslight_util.set_log import init_log
-from fosslight_util.output_format import check_output_formats  # , write_output_file
+from fosslight_util.output_format import check_output_formats_v2  # , write_output_file
 from ._parsing_scanoss_file import parsing_scanResult  # scanoss
 from ._parsing_scanoss_file import parsing_extraInfo  # scanoss
 import shutil
@@ -29,7 +29,7 @@ def get_scanoss_extra_info(scanned_result: dict) -> list:
     return parsing_extraInfo(scanned_result)
 
 
-def run_scanoss_py(path_to_scan: str, output_file_name: str = "", format: str = "", called_by_cli: bool = False,
+def run_scanoss_py(path_to_scan: str, output_file_name: str = "", format: list = [], called_by_cli: bool = False,
                    write_json_file: bool = False, num_threads: int = -1, path_to_exclude: list = []) -> list:
     """
     Run scanoss.py for the given path.
@@ -41,7 +41,7 @@ def run_scanoss_py(path_to_scan: str, output_file_name: str = "", format: str =
     :param write_json_file: if requested, keep the raw files.
     :return scanoss_file_list: list of ScanItem (scanned result by files).
     """
-    success, msg, output_path, output_files, output_extensions = check_output_formats(output_file_name, format)
+    success, msg, output_path, output_files, output_extensions, formats = check_output_formats_v2(output_file_name, format)
 
     if not called_by_cli:
         global logger

tests/cli_test.py

@@ -25,8 +25,8 @@ def main():
 
     logger, result_item = init_log(os.path.join(output_dir, "fosslight_log_"+_start_time+".txt"))
 
-    ret = run_scan(path_to_find_bin, fosslight_report_name, True, -1, True, True, "", False)
-    ret_scanoss = run_scanoss_py(path_to_find_bin, fosslight_report_name, "", False, True, -1)
+    ret = run_scan(path_to_find_bin, fosslight_report_name, True, -1, True, True, [], False)
+    ret_scanoss = run_scanoss_py(path_to_find_bin, fosslight_report_name, [], False, True, -1)
 
     logger.warning("[Scan] Result: %s" % (ret[0]))
     logger.warning("[Scan] Result_msg: %s" % (ret[1]))