diff --git a/requirements.txt b/requirements.txt index 8a80bc1..3b3e983 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ pyparsing -scancode-toolkit>=32.0.2,==32.0.* +scancode-toolkit>=32.2.0 scanoss XlsxWriter -fosslight_util>=2.0.0 +fosslight_util>=2.1.0 PyYAML wheel>=0.38.1 intbitset diff --git a/src/fosslight_source/_parsing_scanoss_file.py b/src/fosslight_source/_parsing_scanoss_file.py index aabf9b4..1ef93ff 100644 --- a/src/fosslight_source/_parsing_scanoss_file.py +++ b/src/fosslight_source/_parsing_scanoss_file.py @@ -45,6 +45,7 @@ def parsing_scanResult(scanoss_report: dict, path_to_scan: str = "", path_to_exc if any(os.path.commonpath([abs_file_path, exclude_path]) == exclude_path for exclude_path in abs_path_to_exclude): continue result_item = SourceItem(file_path) + if 'id' in findings[0]: if "none" == findings[0]['id']: continue diff --git a/src/fosslight_source/_scan_item.py b/src/fosslight_source/_scan_item.py index 46c3266..6d894b7 100644 --- a/src/fosslight_source/_scan_item.py +++ b/src/fosslight_source/_scan_item.py @@ -7,7 +7,7 @@ import logging import re import fosslight_util.constant as constant -from fosslight_util.oss_item import FileItem, OssItem +from fosslight_util.oss_item import FileItem, OssItem, get_checksum_sha1 logger = logging.getLogger(constant.LOGGER_NAME) replace_word = ["-only", "-old-style", "-or-later", "licenseref-scancode-", "licenseref-"] @@ -42,6 +42,8 @@ def __init__(self, value: str) -> None: self.oss_name = "" self.oss_version = "" + self.checksum = get_checksum_sha1(value) + def __del__(self) -> None: pass diff --git a/src/fosslight_source/cli.py b/src/fosslight_source/cli.py index 48e7425..7e824e1 100755 --- a/src/fosslight_source/cli.py +++ b/src/fosslight_source/cli.py @@ -5,6 +5,7 @@ import sys import os +import platform import warnings import logging from datetime import datetime @@ -13,7 +14,7 @@ from fosslight_util.timer_thread import TimerThread from ._help import print_version, print_help_msg_source_scanner from ._license_matched import get_license_list_to_print -from fosslight_util.output_format import check_output_formats, write_output_file +from fosslight_util.output_format import check_output_formats_v2, write_output_file from fosslight_util.correct import correct_with_yaml from .run_scancode import run_scan from .run_scanoss import run_scanoss_py @@ -49,7 +50,7 @@ def main() -> None: write_json_file = False output_file_name = "" print_matched_text = False - formats = "" + formats = [] selected_scanner = "" correct_mode = True @@ -145,7 +146,8 @@ def create_report_file( selected_scanner: str, need_license: bool = False, output_path: str = "", output_files: list = [], output_extensions: list = [], correct_mode: bool = True, - correct_filepath: str = "", path_to_scan: str = "", path_to_exclude: list = [] + correct_filepath: str = "", path_to_scan: str = "", path_to_exclude: list = [], + formats: list = [] ) -> 'ScannerItem': """ Create report files for given scanned result. @@ -168,12 +170,33 @@ def create_report_file( # If -o does not contains file name, set default name while len(output_files) < len(output_extensions): output_files.append(None) + to_remove = [] # elements of spdx format on windows that should be removed for i, output_extension in enumerate(output_extensions): if output_files[i] is None or output_files[i] == "": - if output_extension == _json_ext: - output_files[i] = f"fosslight_opossum_src_{_start_time}" + if formats: + if formats[i].startswith('spdx'): + if platform.system() != 'Windows': + output_files[i] = f"fosslight_spdx_src_{_start_time}" + else: + logger.warning('spdx format is not supported on Windows. Please remove spdx from format.') + to_remove.append(i) + else: + if output_extension == _json_ext: + output_files[i] = f"fosslight_opossum_src_{_start_time}" + else: + output_files[i] = f"fosslight_report_src_{_start_time}" else: - output_files[i] = f"fosslight_report_src_{_start_time}" + if output_extension == _json_ext: + output_files[i] = f"fosslight_opossum_src_{_start_time}" + else: + output_files[i] = f"fosslight_report_src_{_start_time}" + for index in sorted(to_remove, reverse=True): + # remove elements of spdx format on windows + del output_files[index] + del output_extensions[index] + del formats[index] + if len(output_extensions) < 1: + sys.exit(0) if not correct_filepath: correct_filepath = path_to_scan @@ -219,10 +242,10 @@ def create_report_file( combined_paths_and_files = [os.path.join(output_path, file) for file in output_files] results = [] - for combined_path_and_file, output_extension in zip(combined_paths_and_files, output_extensions): + for combined_path_and_file, output_extension, output_format in zip(combined_paths_and_files, output_extensions, formats): # if need_license and output_extension == _json_ext and "scanoss_reference" in sheet_list: # del sheet_list["scanoss_reference"] - results.append(write_output_file(combined_path_and_file, output_extension, scan_item, extended_header, "")) + results.append(write_output_file(combined_path_and_file, output_extension, scan_item, extended_header, "", output_format)) for success, msg, result_file in results: if success: logger.info(f"Output file: {result_file}") @@ -295,8 +318,9 @@ def run_scanners( license_list = [] spdx_downloads = {} result_log = {} + scan_item = [] - success, msg, output_path, output_files, output_extensions = check_output_formats(output_file_name, formats) + success, msg, output_path, output_files, output_extensions, formats = check_output_formats_v2(output_file_name, formats) logger, result_log = init_log(os.path.join(output_path, f"fosslight_log_src_{start_time}.txt"), True, logging.INFO, logging.DEBUG, PKG_NAME, path_to_scan, path_to_exclude) @@ -319,7 +343,7 @@ def run_scanners( merged_result = merge_results(scancode_result, scanoss_result, spdx_downloads) scan_item = create_report_file(start_time, merged_result, license_list, scanoss_result, selected_scanner, print_matched_text, output_path, output_files, output_extensions, correct_mode, - correct_filepath, path_to_scan, path_to_exclude) + correct_filepath, path_to_scan, path_to_exclude, formats) else: print_help_msg_source_scanner() result_log[RESULT_KEY] = "Unsupported scanner" diff --git a/src/fosslight_source/run_scancode.py b/src/fosslight_source/run_scancode.py index 61c01c2..1f03e3d 100755 --- a/src/fosslight_source/run_scancode.py +++ b/src/fosslight_source/run_scancode.py @@ -14,7 +14,7 @@ from fosslight_util.set_log import init_log from ._parsing_scancode_file_item import parsing_file_item from ._parsing_scancode_file_item import get_error_from_header -from fosslight_util.output_format import check_output_formats +from fosslight_util.output_format import check_output_formats_v2 from fosslight_binary.binary_analysis import check_binary from typing import Tuple @@ -46,7 +46,7 @@ def run_scan( if not correct_filepath: correct_filepath = path_to_scan - success, msg, output_path, output_files, output_extensions = check_output_formats(output_file_name, formats) + success, msg, output_path, output_files, output_extensions, formats = check_output_formats_v2(output_file_name, formats) if success: if output_path == "": # if json output with _write_json_file not used, output_path won't be needed. output_path = os.getcwd() diff --git a/src/fosslight_source/run_scanoss.py b/src/fosslight_source/run_scanoss.py index 2b5a441..ea89d50 100755 --- a/src/fosslight_source/run_scanoss.py +++ b/src/fosslight_source/run_scanoss.py @@ -11,7 +11,7 @@ from datetime import datetime import fosslight_util.constant as constant from fosslight_util.set_log import init_log -from fosslight_util.output_format import check_output_formats # , write_output_file +from fosslight_util.output_format import check_output_formats_v2 # , write_output_file from ._parsing_scanoss_file import parsing_scanResult # scanoss from ._parsing_scanoss_file import parsing_extraInfo # scanoss import shutil @@ -29,7 +29,7 @@ def get_scanoss_extra_info(scanned_result: dict) -> list: return parsing_extraInfo(scanned_result) -def run_scanoss_py(path_to_scan: str, output_file_name: str = "", format: str = "", called_by_cli: bool = False, +def run_scanoss_py(path_to_scan: str, output_file_name: str = "", format: list = [], called_by_cli: bool = False, write_json_file: bool = False, num_threads: int = -1, path_to_exclude: list = []) -> list: """ Run scanoss.py for the given path. @@ -41,7 +41,7 @@ def run_scanoss_py(path_to_scan: str, output_file_name: str = "", format: str = :param write_json_file: if requested, keep the raw files. :return scanoss_file_list: list of ScanItem (scanned result by files). """ - success, msg, output_path, output_files, output_extensions = check_output_formats(output_file_name, format) + success, msg, output_path, output_files, output_extensions, formats = check_output_formats_v2(output_file_name, format) if not called_by_cli: global logger diff --git a/tests/cli_test.py b/tests/cli_test.py index e35a8c7..5d68af7 100755 --- a/tests/cli_test.py +++ b/tests/cli_test.py @@ -25,8 +25,8 @@ def main(): logger, result_item = init_log(os.path.join(output_dir, "fosslight_log_"+_start_time+".txt")) - ret = run_scan(path_to_find_bin, fosslight_report_name, True, -1, True, True, "", False) - ret_scanoss = run_scanoss_py(path_to_find_bin, fosslight_report_name, "", False, True, -1) + ret = run_scan(path_to_find_bin, fosslight_report_name, True, -1, True, True, [], False) + ret_scanoss = run_scanoss_py(path_to_find_bin, fosslight_report_name, [], False, True, -1) logger.warning("[Scan] Result: %s" % (ret[0])) logger.warning("[Scan] Result_msg: %s" % (ret[1]))