From dd429cd9d037e8f8d337e678d535c7f198608fdf Mon Sep 17 00:00:00 2001 From: Jaekwon Bang Date: Tue, 8 Oct 2024 16:03:10 +0900 Subject: [PATCH] Apply simple mode --- src/fosslight_binary/_simple_mode.py | 111 ++++++++++++++++++++++++ src/fosslight_binary/binary_analysis.py | 49 +++++++---- src/fosslight_binary/cli.py | 2 +- 3 files changed, 143 insertions(+), 19 deletions(-) create mode 100644 src/fosslight_binary/_simple_mode.py diff --git a/src/fosslight_binary/_simple_mode.py b/src/fosslight_binary/_simple_mode.py new file mode 100644 index 0000000..5950d3c --- /dev/null +++ b/src/fosslight_binary/_simple_mode.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# FOSSLight Binary analysis script +# Copyright (c) 2024 LG Electronics Inc. +# SPDX-License-Identifier: Apache-2.0 +import os +import re +import logging +from datetime import datetime +import fosslight_util.constant as constant +from fosslight_util.write_txt import write_txt_file +from fosslight_util.download import compression_extension +import fosslight_binary.binary_analysis as fl_bin +from fosslight_util.set_log import init_log + +_REMOVE_FILE_EXTENSION_SIMPLE = ['ttf', 'otf', 'png', 'gif', 'jpg', 'bmp', 'jpeg'] +logger = logging.getLogger(constant.LOGGER_NAME) + + +def exclude_bin_for_simple_mode(binary_list): + bin_list = [] + compressed_list = [] + compressed_ext_wo_dot = [ext.lstrip('.') for ext in compression_extension] + + for bin in binary_list: + file_lower_case = bin.bin_name_with_path.lower() + extension = os.path.splitext(file_lower_case)[1][1:].strip() + + for compress_ext in compressed_ext_wo_dot: + if extension == compress_ext: + compressed_list.append(bin.bin_name_with_path) + + remove_file_ext_list = _REMOVE_FILE_EXTENSION_SIMPLE + compressed_ext_wo_dot + if any(extension == remove_ext for remove_ext in remove_file_ext_list): + continue + if not (re.search(r".*source\.jar", bin.bin_name_with_path.lower()) or bin.exclude): + continue + + bin_list.append(bin.bin_name_with_path) + + return compressed_list, bin_list + + +def convert_list_to_str(input_list): + output_text = '\n'.join(map(str, input_list)) + return output_text + + +def check_output_path(output='', start_time): + compressed_list_txt = "" + simple_bin_list_txt = "" + output_path = "" + + if output != "": + if not os.path.isdir(output) and output.endswith('.txt'): + output_path = os.path.dirname(output) + basename = os.path.basename(output) + basename_file, _ = os.path.splitext(basename) + compressed_list_txt = f"{basename_file}_compressed_list.txt" + simple_bin_list_txt = f"{basename_file}.txt" + else: + output_path = output + compressed_list_txt = f"compressed_list_{start_time}.txt" + simple_bin_list_txt = f"binary_list_{start_time}.txt" + else: + compressed_list_txt = f"compressed_list_{start_time}.txt" + simple_bin_list_txt = f"binary_list_{start_time}.txt" + + if output_path == "": + output_path = os.getcwd() + else: + output_path = os.path.abspath(output_path) + + compressed_list_txt = os.path.join(output_path, compressed_list_txt) + simple_bin_list_txt = os.path.join(output_path, simple_bin_list_txt) + + return output_path, compressed_list_txt, simple_bin_list_txt + + +def init_simple(output_file_name, pkg_name, start_time): + global logger + + _result_log = { + "Tool Info": pkg_name, + "Mode": "Simple mode" + } + + output_path, compressed_list_txt, simple_bin_list_txt = check_output_path(output_file_name, start_time) + + log_file = os.path.join(output_path, f"fosslight_log_bin_{start_time}.txt") + logger, _result_log = init_log(log_file, True, logging.INFO, logging.DEBUG, + pkg_name) + + return _result_log, compressed_list_txt, simple_bin_list_txt + + +def print_simple_mode(compressed_list_txt, simple_bin_list_txt, compressed_list, bin_list): + if compressed_list: + success, error = write_txt_file(compressed_list_txt, convert_list_to_str(compressed_list)) + if not success: + logger.info(f"Error to write compressed list file for simple mode : {error}") + if bin_list: + success, error = write_txt_file(simple_bin_list_txt, convert_list_to_str(bin_list)) + if not success: + logger.info(f"Error to write binary list file for simple mode : {error}") + return success + + +def filter_binary(bin_list): + compressed_list, bin_list = exclude_bin_for_simple_mode(bin_list) + return compressed_list, bin_list diff --git a/src/fosslight_binary/binary_analysis.py b/src/fosslight_binary/binary_analysis.py index d60165f..f24dad6 100755 --- a/src/fosslight_binary/binary_analysis.py +++ b/src/fosslight_binary/binary_analysis.py @@ -13,10 +13,11 @@ import stat from fosslight_util.set_log import init_log import fosslight_util.constant as constant -from fosslight_util.output_format import check_output_formats, write_output_file -from ._binary_dao import get_oss_info_from_db +from fosslight_util.output_format import check_output_formats_v2, write_output_file from ._binary import BinaryItem, TLSH_CHECKSUM_NULL +from ._binary_dao import get_oss_info_from_db from ._jar_analysis import analyze_jar_file, merge_binary_list +from ._simple_mode import print_simple_mode, filter_binary, init_simple from fosslight_util.correct import correct_with_yaml from fosslight_util.oss_item import ScannerItem import hashlib @@ -40,7 +41,7 @@ _REMOVE_DIR = [os.path.sep + dir_name + os.path.sep for dir_name in _REMOVE_DIR] _error_logs = [] _root_path = "" -_start_time = "" +start_time = "" windows = False BYTES = 2048 BIN_EXT_HEADER = {'BIN_FL_Binary': ['ID', 'Binary Path', 'OSS Name', @@ -71,18 +72,14 @@ def get_checksum_and_tlsh(bin_with_path): def init(path_to_find_bin, output_file_name, formats, path_to_exclude=[]): - global _root_path, logger, _start_time + global logger _json_ext = ".json" - _start_time = datetime.now().strftime('%y%m%d_%H%M') _result_log = { - "Tool Info": PKG_NAME + "Tool Info": PKG_NAME, + "Mode": "Normal Mode" } - _root_path = path_to_find_bin - if not path_to_find_bin.endswith(os.path.sep): - _root_path += os.path.sep - success, msg, output_path, output_files, output_extensions = check_output_formats(output_file_name, formats) if success: @@ -96,16 +93,16 @@ def init(path_to_find_bin, output_file_name, formats, path_to_exclude=[]): for i, output_extension in enumerate(output_extensions): if output_files[i] is None or output_files[i] == "": if output_extension == _json_ext: - output_files[i] = f"fosslight_opossum_bin_{_start_time}" + output_files[i] = f"fosslight_opossum_bin_{start_time}" else: - output_files[i] = f"fosslight_report_bin_{_start_time}" + output_files[i] = f"fosslight_report_bin_{start_time}" combined_paths_and_files = [os.path.join(output_path, file) for file in output_files] else: logger.error(f"Format error - {msg}") sys.exit(1) - log_file = os.path.join(output_path, f"fosslight_log_bin_{_start_time}.txt") + log_file = os.path.join(output_path, f"fosslight_log_bin_{start_time}.txt") logger, _result_log = init_log(log_file, True, logging.INFO, logging.DEBUG, PKG_NAME, path_to_find_bin, path_to_exclude) @@ -161,9 +158,19 @@ def get_file_list(path_to_find, abs_path_to_exclude): def find_binaries(path_to_find_bin, output_dir, formats, dburl="", simple_mode=False, correct_mode=True, correct_filepath="", path_to_exclude=[]): + global start_time, _root_path - _result_log, result_reports, output_extensions = init( - path_to_find_bin, output_dir, formats, path_to_exclude) + start_time = datetime.now().strftime('%y%m%d_%H%M') + + _root_path = path_to_find_bin + if not path_to_find_bin.endswith(os.path.sep): + _root_path += os.path.sep + + if simple_mode: + _result_log, compressed_list_txt, simple_bin_list_txt = init_simple(output_dir, PKG_NAME, start_time) + else: + _result_log, result_reports, output_extensions = init( + path_to_find_bin, output_dir, formats, path_to_exclude) total_bin_cnt = 0 total_file_cnt = 0 @@ -191,9 +198,15 @@ def find_binaries(path_to_find_bin, output_dir, formats, dburl="", simple_mode=F exit=True) total_bin_cnt = len(return_list) if simple_mode: - bin_list = [bin.bin_name_with_path for bin in return_list] + try: + compressed_list, bin_list = filter_binary(return_list) + success = print_simple_mode(compressed_list_txt, simple_bin_list_txt, compressed_list, bin_list) + except Exception as ex: + error_occured(error_msg=f"Failed to run simple mode: {ex}", + result_log=_result_log, + exit=True) else: - scan_item = ScannerItem(PKG_NAME, _start_time) + scan_item = ScannerItem(PKG_NAME, start_time) scan_item.set_cover_pathinfo(path_to_find_bin, path_to_exclude) try: # Run OWASP Dependency-check @@ -307,7 +320,7 @@ def print_result_log(success=True, result_log={}, file_cnt="", bin_file_cnt="", if "Running time" in result_log: start_time = result_log["Running time"] else: - start_time = _start_time + start_time = start_time result_log["Running time"] = start_time + " ~ " + \ datetime.now().strftime('%Y%m%d_%H%M%S') result_log["Execution result"] = 'Success' if success else 'Error occurred' diff --git a/src/fosslight_binary/cli.py b/src/fosslight_binary/cli.py index 2086d34..ae846b4 100644 --- a/src/fosslight_binary/cli.py +++ b/src/fosslight_binary/cli.py @@ -49,7 +49,7 @@ def main(): print_package_version(_PKG_NAME, "FOSSLight Binary Scanner Version:") sys.exit(0) - if args.simple: + if args.simple: # -s option simple_mode = True if args.path: # -p option