From db23456a2eca2bb30f5c394eac4d1a1bcee73033 Mon Sep 17 00:00:00 2001 From: Jaekwon Bang Date: Wed, 19 Feb 2025 16:31:01 +0900 Subject: [PATCH] Import exclude path fct. from FL util --- requirements.txt | 2 +- src/fosslight_prechecker/_add.py | 14 +++++----- src/fosslight_prechecker/_precheck.py | 38 +++++++++++---------------- src/fosslight_prechecker/cli.py | 2 +- 4 files changed, 26 insertions(+), 30 deletions(-) diff --git a/requirements.txt b/requirements.txt index f5d244a..158610c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,5 @@ binaryornot requests reuse==1.1.2 PyYAML -fosslight_util==2.* +fosslight_util>=2.1.10 jinja2 diff --git a/src/fosslight_prechecker/_add.py b/src/fosslight_prechecker/_add.py index 4096889..5fade6a 100644 --- a/src/fosslight_prechecker/_add.py +++ b/src/fosslight_prechecker/_add.py @@ -15,6 +15,7 @@ from fosslight_util.spdx_licenses import get_spdx_licenses_json, get_license_from_nick from fosslight_util.parsing_yaml import find_sbom_yaml_files, parsing_yml from fosslight_util.output_format import check_output_format +from fosslight_util.exclude import excluding_files from datetime import datetime from fosslight_prechecker._precheck import precheck_for_project, precheck_for_files, dump_error_msg, \ get_path_to_find, DEFAULT_EXCLUDE_EXTENSION_FILES @@ -402,7 +403,8 @@ def add_content( input_copyright: str = "", input_dl_url: str = "", output_path: str = "", - need_log_file: bool = True + need_log_file: bool = True, + exclude_path: list = [] ) -> None: global _result_log, spdx_licenses _check_only_file_mode = False @@ -417,6 +419,9 @@ def add_content( else: output_path = os.path.abspath(output_path) + user_exclude_path = excluding_files(exclude_path, path_to_find) + abs_path_to_exclude = [os.path.abspath(path) for path in user_exclude_path] + now = datetime.now().strftime('%y%m%d_%H%M') logger, _result_log = init_log(os.path.join(output_path, f"fosslight_log_pre_{now}.txt"), need_log_file, logging.INFO, logging.DEBUG, PKG_NAME, path_to_find) @@ -440,7 +445,6 @@ def add_content( except Exception as ex: dump_error_msg(f"Error access to get_spdx_licenses_json : {ex}") - # File Only mode (-f option) if _check_only_file_mode: main_parser = reuse_parser() missing_license_list, missing_copyright_list, project = precheck_for_files(path_to_find, file_to_check_list) @@ -468,17 +472,15 @@ def add_content( # Add Download URL if input_dl_url: add_dl_url_into_file(main_parser, project, path_to_find, input_dl_url, file_to_check_list) - - # Path mode (-p option) else: # Download license text file of OSS-pkg-info.yaml download_oss_info_license(path_to_find, input_license) # Get missing license / copyright file list - missing_license, missing_copyright, _, project, prj_report = precheck_for_project(path_to_find) + missing_license, missing_copyright, _, project, prj_report = precheck_for_project(path_to_find, abs_path_to_exclude) # Get total files except excluded file - total_files_excluded = get_total_file_list(path_to_find, prj_report, DEFAULT_EXCLUDE_EXTENSION_FILES) + total_files_excluded = get_total_file_list(path_to_find, prj_report, DEFAULT_EXCLUDE_EXTENSION_FILES, abs_path_to_exclude) skip_files = sorted(set(total_files_excluded) - set(missing_license) - set(missing_copyright)) logger.info(f"\n# File list that have both license and copyright : {len(skip_files)} / {len(total_files_excluded)}") diff --git a/src/fosslight_prechecker/_precheck.py b/src/fosslight_prechecker/_precheck.py index cb9031f..7a613e5 100755 --- a/src/fosslight_prechecker/_precheck.py +++ b/src/fosslight_prechecker/_precheck.py @@ -16,6 +16,7 @@ import fosslight_util.constant as constant from fosslight_util.set_log import init_log from fosslight_util.timer_thread import TimerThread +from fosslight_util.exclude import excluding_files from reuse import report from reuse.project import Project from reuse.report import ProjectReport @@ -26,7 +27,6 @@ is_windows = platform.system() == 'Windows' REUSE_CONFIG_FILE = ".reuse/dep5" DEFAULT_EXCLUDE_EXTENSION_FILES = [] # Exclude files from reuse -user_exclude_list = [] # Exclude paths from checking _turn_on_exclude_config = True _check_only_file_mode = False error_items = [] @@ -88,7 +88,7 @@ def exclude_git_related_files(path: str) -> None: logger.warning(f"Error to get git related files : {ex}") -def find_oss_pkg_info_and_exclude_file(path: str) -> List[str]: +def find_oss_pkg_info_and_exclude_file(path: str, abs_path_to_exclude: List) -> List[str]: global DEFAULT_EXCLUDE_EXTENSION_FILES oss_pkg_info = [] git_present = shutil.which("git") @@ -98,13 +98,13 @@ def find_oss_pkg_info_and_exclude_file(path: str) -> List[str]: try: for root, dirs, files in os.walk(path): - if os.path.abspath(root) in user_exclude_list: + if os.path.abspath(root) in abs_path_to_exclude: continue for dir in dirs: # For hidden folders dir_path = os.path.join(root, dir) dir_abs_path = os.path.abspath(dir_path) - if any(os.path.commonpath([dir_abs_path, exclude_path]) == exclude_path for exclude_path in user_exclude_list): + if any(os.path.commonpath([dir_abs_path, exclude_path]) == exclude_path for exclude_path in abs_path_to_exclude): continue if dir.startswith("."): all_files_in_dir = [os.path.join(root, dir, file) @@ -116,7 +116,7 @@ def find_oss_pkg_info_and_exclude_file(path: str) -> List[str]: for file in files: file_path = os.path.join(root, file) file_abs_path = os.path.abspath(file_path) - if any(os.path.commonpath([file_abs_path, exclude_path]) == exclude_path for exclude_path in user_exclude_list): + if any(os.path.commonpath([file_abs_path, exclude_path]) == exclude_path for exclude_path in abs_path_to_exclude): continue file_lower_case = file.lower() file_rel_path = os.path.relpath(file_path, path) @@ -243,11 +243,11 @@ def precheck_for_files(path: str, files: List[str]) -> Tuple[List[str], List[str return missing_license_list, missing_copyright_list, prj -def precheck_for_project(path_to_find: str) -> Tuple[List[str], List[str], List[str], Project, ProjectReport]: +def precheck_for_project(path_to_find: str, abs_path_to_exclude: List) -> Tuple[List[str], List[str], List[str], Project, ProjectReport]: missing_license = [] missing_copyright = [] - oss_pkg_info_files = find_oss_pkg_info_and_exclude_file(path_to_find) + oss_pkg_info_files = find_oss_pkg_info_and_exclude_file(path_to_find, abs_path_to_exclude) if _turn_on_exclude_config: need_rollback, temp_file_name, temp_dir_name = create_reuse_dep5_file(path_to_find) @@ -259,14 +259,14 @@ def precheck_for_project(path_to_find: str) -> Tuple[List[str], List[str], List[ missing_license = [str(sub) for sub in set(report.files_without_licenses)] if not path_to_find.endswith(f"{os.sep}"): path_to_find += f"{os.sep}" - missing_license = filter_missing_list(missing_license) + missing_license = filter_missing_list(missing_license, abs_path_to_exclude) missing_license = [sub.replace(path_to_find, '', 1) for sub in missing_license] # File list that missing copyright text missing_copyright = [str(sub) for sub in set(report.files_without_copyright)] if not path_to_find.endswith(f"{os.sep}"): path_to_find += f"{os.sep}" - missing_copyright = filter_missing_list(missing_copyright) + missing_copyright = filter_missing_list(missing_copyright, abs_path_to_exclude) missing_copyright = [sub.replace(path_to_find, '', 1) for sub in missing_copyright] except Exception as ex: @@ -277,11 +277,11 @@ def precheck_for_project(path_to_find: str) -> Tuple[List[str], List[str], List[ return missing_license, missing_copyright, oss_pkg_info_files, project, report -def filter_missing_list(missing_list: List[str]) -> List[str]: +def filter_missing_list(missing_list: List[str], abs_path_to_exclude: List) -> List[str]: filtered_list = [] for file in missing_list: abs_path = os.path.abspath(file) - if not any(os.path.commonpath([abs_path, path]) == path for path in user_exclude_list): + if not any(os.path.commonpath([abs_path, path]) == path for path in abs_path_to_exclude): filtered_list.append(file) return filtered_list @@ -339,14 +339,6 @@ def get_path_to_find(target_path: str, _check_only_file_mode: bool) -> Tuple[str return path_to_find, file_to_check_list, _check_only_file_mode -def set_exclude_list(path_to_find: str, exclude_path: List[str]): - global user_exclude_list - - for path in exclude_path: - if path.strip != "": - user_exclude_list.append(os.path.abspath(os.path.join(path_to_find, path))) - - def run_lint( target_path: str, disable: bool, @@ -357,6 +349,7 @@ def run_lint( ) -> None: global _turn_on_exclude_config, _check_only_file_mode, _start_time + abs_path_to_exclude = [] file_to_check_list = [] _exit_code = 0 path_to_find = "" @@ -371,7 +364,8 @@ def run_lint( dump_error_msg(f"Error - locale : {ex}") path_to_find, file_to_check_list, _check_only_file_mode = get_path_to_find(target_path, _check_only_file_mode) - set_exclude_list(path_to_find, exclude_path) + user_exclude_path = excluding_files(exclude_path, path_to_find) + abs_path_to_exclude = [os.path.abspath(path) for path in user_exclude_path] result_file, output_path, output_extension = create_result_file(output_file_name, format, _start_time) init(path_to_find, output_path, file_to_check_list, need_log_file, exclude_path) @@ -389,7 +383,7 @@ def run_lint( if _check_only_file_mode: license_missing_files, copyright_missing_files, project = precheck_for_files(path_to_find, file_to_check_list) else: - license_missing_files, copyright_missing_files, oss_pkg_info, project, report = precheck_for_project(path_to_find) + license_missing_files, copyright_missing_files, oss_pkg_info, project, report = precheck_for_project(path_to_find, abs_path_to_exclude) if need_log_file: timer.stop = True @@ -404,7 +398,7 @@ def run_lint( file_to_check_list, error_items, DEFAULT_EXCLUDE_EXTENSION_FILES, - user_exclude_list) + abs_path_to_exclude) success, exit_code = write_result_file(result_file, output_extension, _exit_code, result_item, _result_log, project, path_to_find) diff --git a/src/fosslight_prechecker/cli.py b/src/fosslight_prechecker/cli.py index aa933dc..8f1a454 100644 --- a/src/fosslight_prechecker/cli.py +++ b/src/fosslight_prechecker/cli.py @@ -21,7 +21,7 @@ def run_main(mode: str, path, output, format, no_log, disable, copyright, licens if mode == "lint": run_lint(path, disable, output, format, no_log, exclude_path) elif mode == "add": - add_content(path, license, copyright, dl_url, output, no_log) + add_content(path, license, copyright, dl_url, output, no_log, exclude_path) elif mode == "convert": convert_report(path, output, format, no_log) else: