From 56a7921df43c07dcba707ace7aa60ed02c2a3360 Mon Sep 17 00:00:00 2001 From: Wonjae Park Date: Fri, 10 Jan 2025 16:32:44 +0900 Subject: [PATCH 1/5] Fix exclude error Signed-off-by: Wonjae Park --- src/fosslight_source/cli.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/fosslight_source/cli.py b/src/fosslight_source/cli.py index ab59898..6a54219 100755 --- a/src/fosslight_source/cli.py +++ b/src/fosslight_source/cli.py @@ -12,6 +12,7 @@ import fosslight_util.constant as constant from fosslight_util.set_log import init_log from fosslight_util.timer_thread import TimerThread +from fosslight_util.exclude import excluding_files from ._help import print_version, print_help_msg_source_scanner from ._license_matched import get_license_list_to_print from fosslight_util.output_format import check_output_formats_v2, write_output_file @@ -261,7 +262,8 @@ def create_report_file( return scan_item -def merge_results(scancode_result: list = [], scanoss_result: list = [], spdx_downloads: dict = {}) -> list: +def merge_results(scancode_result: list = [], scanoss_result: list = [], spdx_downloads: dict = {}, excluded_file_list=[]) -> list: + """ Merge scanner results and spdx parsing result. :param scancode_result: list of scancode results in SourceItem. @@ -285,6 +287,11 @@ def merge_results(scancode_result: list = [], scanoss_result: list = [], spdx_do new_result_item.download_location = download_location scancode_result.append(new_result_item) + for i in range(len(scancode_result) - 1, -1, -1): # Iterate from last to first + item_path = scancode_result[i].source_name_or_path # Assuming SourceItem has 'file_path' attribute + if item_path in excluded_file_list: + del scancode_result[i] # Delete matching item + for item in scancode_result: item.set_oss_item() @@ -329,6 +336,7 @@ def run_scanners( logger, result_log = init_log(os.path.join(output_path, f"fosslight_log_src_{start_time}.txt"), True, logging.INFO, logging.DEBUG, PKG_NAME, path_to_scan, path_to_exclude) + excluded_file_list = excluding_files(path_to_exclude, path_to_scan) if '.xlsx' not in output_extensions and print_matched_text: logger.warning("-m option is only available for excel.") @@ -345,7 +353,7 @@ def run_scanners( path_to_exclude) if selected_scanner in SCANNER_TYPE: spdx_downloads = get_spdx_downloads(path_to_scan, path_to_exclude) - merged_result = merge_results(scancode_result, scanoss_result, spdx_downloads) + merged_result = merge_results(scancode_result, scanoss_result, spdx_downloads, excluded_file_list) scan_item = create_report_file(start_time, merged_result, license_list, scanoss_result, selected_scanner, print_matched_text, output_path, output_files, output_extensions, correct_mode, correct_filepath, path_to_scan, path_to_exclude, formats) From f14375a84a412a4abab850cbf3ac60292f063be7 Mon Sep 17 00:00:00 2001 From: Wonjae Park Date: Thu, 23 Jan 2025 10:46:16 +0900 Subject: [PATCH 2/5] move excluding code in to create_report_file Signed-off-by: Wonjae Park --- src/fosslight_source/cli.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/fosslight_source/cli.py b/src/fosslight_source/cli.py index 6a54219..c5f88ff 100755 --- a/src/fosslight_source/cli.py +++ b/src/fosslight_source/cli.py @@ -148,7 +148,7 @@ def create_report_file( output_path: str = "", output_files: list = [], output_extensions: list = [], correct_mode: bool = True, correct_filepath: str = "", path_to_scan: str = "", path_to_exclude: list = [], - formats: list = [] + formats: list = [], excluded_file_list = [] ) -> 'ScannerItem': """ Create report files for given scanned result. @@ -220,6 +220,12 @@ def create_report_file( if merged_result: sheet_list = {} + # Remove results that are in excluding file list + for i in range(len(merged_result) - 1, -1, -1): # Iterate from last to first + item_path = merged_result[i].source_name_or_path # Assuming SourceItem has 'file_path' attribute + if item_path in excluded_file_list: + del merged_result[i] # Delete matching item + scan_item.append_file_items(merged_result, PKG_NAME) if selected_scanner == 'scanoss': @@ -262,7 +268,7 @@ def create_report_file( return scan_item -def merge_results(scancode_result: list = [], scanoss_result: list = [], spdx_downloads: dict = {}, excluded_file_list=[]) -> list: +def merge_results(scancode_result: list = [], scanoss_result: list = [], spdx_downloads: dict = {}) -> list: """ Merge scanner results and spdx parsing result. @@ -287,11 +293,6 @@ def merge_results(scancode_result: list = [], scanoss_result: list = [], spdx_do new_result_item.download_location = download_location scancode_result.append(new_result_item) - for i in range(len(scancode_result) - 1, -1, -1): # Iterate from last to first - item_path = scancode_result[i].source_name_or_path # Assuming SourceItem has 'file_path' attribute - if item_path in excluded_file_list: - del scancode_result[i] # Delete matching item - for item in scancode_result: item.set_oss_item() @@ -353,10 +354,10 @@ def run_scanners( path_to_exclude) if selected_scanner in SCANNER_TYPE: spdx_downloads = get_spdx_downloads(path_to_scan, path_to_exclude) - merged_result = merge_results(scancode_result, scanoss_result, spdx_downloads, excluded_file_list) + merged_result = merge_results(scancode_result, scanoss_result, spdx_downloads) scan_item = create_report_file(start_time, merged_result, license_list, scanoss_result, selected_scanner, print_matched_text, output_path, output_files, output_extensions, correct_mode, - correct_filepath, path_to_scan, path_to_exclude, formats) + correct_filepath, path_to_scan, path_to_exclude, formats, excluded_file_list) else: print_help_msg_source_scanner() result_log[RESULT_KEY] = "Unsupported scanner" From 43936460fa58098043e9a335872ac41efc30c340 Mon Sep 17 00:00:00 2001 From: Wonjae Park Date: Thu, 23 Jan 2025 11:09:27 +0900 Subject: [PATCH 3/5] fix set fl_util requirement 2.1.9 Signed-off-by: Wonjae Park --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index be3938e..8aa7bb4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ pyparsing scanoss<=1.14.0 XlsxWriter -fosslight_util>=2.1.6 +fosslight_util>=2.1.9 PyYAML wheel>=0.38.1 intbitset From 24fe8a44d6bd3cb07d104e3938669553b312f792 Mon Sep 17 00:00:00 2001 From: Wonjae Park Date: Thu, 23 Jan 2025 18:10:09 +0900 Subject: [PATCH 4/5] fix tox error Signed-off-by: Wonjae Park --- src/fosslight_source/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fosslight_source/cli.py b/src/fosslight_source/cli.py index c5f88ff..eb775ef 100755 --- a/src/fosslight_source/cli.py +++ b/src/fosslight_source/cli.py @@ -148,7 +148,7 @@ def create_report_file( output_path: str = "", output_files: list = [], output_extensions: list = [], correct_mode: bool = True, correct_filepath: str = "", path_to_scan: str = "", path_to_exclude: list = [], - formats: list = [], excluded_file_list = [] + formats: list = [], excluded_file_list: list = [] ) -> 'ScannerItem': """ Create report files for given scanned result. @@ -357,7 +357,7 @@ def run_scanners( merged_result = merge_results(scancode_result, scanoss_result, spdx_downloads) scan_item = create_report_file(start_time, merged_result, license_list, scanoss_result, selected_scanner, print_matched_text, output_path, output_files, output_extensions, correct_mode, - correct_filepath, path_to_scan, path_to_exclude, formats, excluded_file_list) + correct_filepath, path_to_scan, path_to_exclude, formats, excluded_file_list) else: print_help_msg_source_scanner() result_log[RESULT_KEY] = "Unsupported scanner" From a2202666a1edda59fef907f09cb91d433ea21c43 Mon Sep 17 00:00:00 2001 From: Wonjae Park Date: Fri, 24 Jan 2025 15:42:07 +0900 Subject: [PATCH 5/5] change FL Util minimum version Signed-off-by: Wonjae Park --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8aa7bb4..78b1bf8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ pyparsing scanoss<=1.14.0 XlsxWriter -fosslight_util>=2.1.9 +fosslight_util>=2.1.10 PyYAML wheel>=0.38.1 intbitset