Skip to content

Commit 133ad4b

Browse files
JustinWonjaeParkJustinWonjaePark
authored
Create run_scanners for api and exclude unwanted outputs (#140)
* Create run_scanners for API Signed-off-by: Wonjae Park <[email protected]> * Remove deprecated code Signed-off-by: Wonjae Park <[email protected]> * Set exclude .m4 files Signed-off-by: Wonjae Park <[email protected]> * Set binary files from ScanCode result Excluded Signed-off-by: Wonjae Park <[email protected]> * Fix requirements Signed-off-by: Wonjae Park <[email protected]> * Remove condition for creating report Signed-off-by: Wonjae Park <[email protected]> --------- Signed-off-by: Wonjae Park <[email protected]>
1 parent 82a4d25 commit 133ad4b

File tree

4 files changed

+71
-53
lines changed

4 files changed

+71
-53
lines changed

‎requirements.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,4 @@ typecode-libmagic
55
fosslight_util>=1.4.28
66
PyYAML
77
wheel>=0.38.1
8+
fosslight_binary

‎src/fosslight_source/_scan_item.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
"aclocal.m4", "configure", "configure.ac",
1515
"depcomp", "compile", "missing", "libtool.m4",
1616
"makefile"]
17+
_exclude_extension = [".m4"]
1718
_exclude_directory = ["test", "tests", "doc", "docs"]
1819
_exclude_directory = [os.path.sep + dir_name +
1920
os.path.sep for dir_name in _exclude_directory]
@@ -104,6 +105,8 @@ def is_exclude_dir(dir_path):
104105
def is_exclude_file(file_path, prev_dir=None, prev_dir_exclude_value=None):
105106
file_path = file_path.lower()
106107
filename = os.path.basename(file_path)
108+
if os.path.splitext(filename)[1] in _exclude_extension:
109+
return True
107110
if filename in _exclude_filename:
108111
return True
109112

‎src/fosslight_source/cli.py‎

Lines changed: 60 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@
3939

4040
def main():
4141
global logger
42-
success = True
4342
_result_log = {}
4443

4544
path_to_scan = os.getcwd()
@@ -50,8 +49,6 @@ def main():
5049
selected_scanner = ""
5150
correct_mode = True
5251

53-
license_list = []
54-
scanoss_result = []
5552
time_out = 120
5653
core = -1
5754

@@ -101,39 +98,11 @@ def main():
10198
timer.setDaemon(True)
10299
timer.start()
103100

104-
_start_time = datetime.now().strftime('%y%m%d_%H%M')
105-
success, msg, output_path, output_file, output_extension = check_output_format(output_file_name, format)
106-
if output_extension != '.xlsx' and output_extension != "" and print_matched_text:
107-
logger.warning("-m option is only available for excel.")
108-
print_matched_text = False
109-
if not success:
110-
logger.error(f"Format error. {msg}")
111-
sys.exit(1)
112-
logger, _result_log = init_log(os.path.join(output_path, f"fosslight_log_src_{_start_time}.txt"),
113-
True, logging.INFO, logging.DEBUG, _PKG_NAME, path_to_scan)
114-
115101
if os.path.isdir(path_to_scan):
116-
scancode_result = []
117-
scanoss_result = []
118-
merged_result = []
119-
spdx_downloads = {}
120-
success = True
121-
122-
if selected_scanner == 'scancode' or selected_scanner == 'all' or selected_scanner == '':
123-
success, _result_log["Scan Result"], scancode_result, license_list = run_scan(path_to_scan, output_file_name,
124-
write_json_file, core, True,
125-
print_matched_text, format, True,
126-
time_out, correct_mode,
127-
correct_filepath)
128-
if selected_scanner == 'scanoss' or selected_scanner == 'all' or selected_scanner == '':
129-
scanoss_result = run_scanoss_py(path_to_scan, output_file_name, format, True, write_json_file)
130-
if selected_scanner not in SCANNER_TYPE:
131-
print_help_msg_source_scanner()
132-
sys.exit(1)
133-
spdx_downloads = get_spdx_downloads(path_to_scan)
134-
merged_result = merge_results(scancode_result, scanoss_result, spdx_downloads)
135-
create_report_file(_start_time, merged_result, license_list, scanoss_result, selected_scanner, print_matched_text,
136-
output_path, output_file, output_extension, correct_mode, correct_filepath, path_to_scan)
102+
result = []
103+
result = run_scanners(path_to_scan, output_file_name, write_json_file, core, True,
104+
print_matched_text, format, time_out, correct_mode, correct_filepath, selected_scanner)
105+
_result_log["Scan Result"] = result[1]
137106

138107
try:
139108
logger.info(yaml.safe_dump(_result_log, allow_unicode=True, sort_keys=True))
@@ -249,5 +218,61 @@ def merge_results(scancode_result=[], scanoss_result=[], spdx_downloads={}):
249218
return scancode_result
250219

251220

221+
def run_scanners(path_to_scan, output_file_name="", _write_json_file=False, num_cores=-1, called_by_cli=True,
222+
print_matched_text=False, format="", time_out=120, correct_mode=True, correct_filepath="",
223+
selected_scanner='all'):
224+
"""
225+
Run Scancode and scanoss.py for the given path.
226+
227+
:param path_to_scan: path of sourcecode to scan.
228+
:param output_file_name: path or file name (with path) for the output.
229+
:param _write_json_file: if requested, keep the raw files.
230+
:param num_cores: number of cores used for scancode scanning.
231+
:param called_by_cli: if not called by cli, initialize logger.
232+
:param print_matched_text: if requested, output matched text (only for scancode).
233+
:param format: output format (excel, csv, opossum).
234+
:return success: success or failure of scancode.
235+
:return _result_log["Scan Result"]:
236+
:return merged_result: merged scan result of scancode and scanoss.
237+
:return license_list: matched text.(only for scancode)
238+
"""
239+
global logger
240+
241+
_start_time = datetime.now().strftime('%y%m%d_%H%M')
242+
scancode_result = []
243+
scanoss_result = []
244+
merged_result = []
245+
spdx_downloads = {}
246+
247+
success, msg, output_path, output_file, output_extension = check_output_format(output_file_name, format)
248+
if output_extension != '.xlsx' and output_extension != "" and print_matched_text:
249+
logger.warning("-m option is only available for excel.")
250+
print_matched_text = False
251+
if not success:
252+
logger.error(f"Format error. {msg}")
253+
sys.exit(1)
254+
logger, _result_log = init_log(os.path.join(output_path, f"fosslight_log_src_{_start_time}.txt"),
255+
True, logging.INFO, logging.DEBUG, _PKG_NAME, path_to_scan)
256+
257+
if selected_scanner == 'scancode' or selected_scanner == 'all' or selected_scanner == '':
258+
success, _result_log["Scan Result"], scancode_result, license_list = run_scan(path_to_scan, output_file_name,
259+
_write_json_file, num_cores, True,
260+
print_matched_text, format, called_by_cli,
261+
time_out, correct_mode, correct_filepath)
262+
if selected_scanner == 'scanoss' or selected_scanner == 'all' or selected_scanner == '':
263+
scanoss_result = run_scanoss_py(path_to_scan, output_file_name, format, True, _write_json_file)
264+
if selected_scanner not in SCANNER_TYPE:
265+
print_help_msg_source_scanner()
266+
sys.exit(1)
267+
268+
spdx_downloads = get_spdx_downloads(path_to_scan)
269+
merged_result = merge_results(scancode_result, scanoss_result, spdx_downloads)
270+
271+
create_report_file(_start_time, merged_result, license_list, scanoss_result, selected_scanner, print_matched_text,
272+
output_path, output_file, output_extension, correct_mode, correct_filepath, path_to_scan)
273+
274+
return success, _result_log["Scan Result"], merged_result, license_list, scanoss_result
275+
276+
252277
if __name__ == '__main__':
253278
main()

‎src/fosslight_source/run_scancode.py‎

Lines changed: 7 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@
1515
from ._parsing_scancode_file_item import parsing_file_item
1616
from ._parsing_scancode_file_item import get_error_from_header
1717
from ._license_matched import get_license_list_to_print
18-
from fosslight_util.output_format import check_output_format, write_output_file
19-
from fosslight_util.correct import correct_with_yaml
18+
from fosslight_util.output_format import check_output_format
19+
from fosslight_binary.binary_analysis import check_binary
2020

2121
logger = logging.getLogger(constant.LOGGER_NAME)
2222
warnings.filterwarnings("ignore", category=FutureWarning)
@@ -99,26 +99,15 @@ def run_scan(path_to_scan, output_file_name="",
9999
success = True
100100
result_list = sorted(
101101
result_list, key=lambda row: (''.join(row.licenses)))
102+
103+
for scan_item in result_list:
104+
if check_binary(os.path.join(path_to_scan, scan_item.file)):
105+
scan_item.exclude = True
106+
102107
sheet_list["SRC_FL_Source"] = [scan_item.get_row_to_print() for scan_item in result_list]
103108
if need_license:
104109
sheet_list["matched_text"] = get_license_list_to_print(license_list)
105110

106-
output_file_without_ext = os.path.join(output_path, output_file)
107-
if not called_by_cli:
108-
if correct_mode:
109-
success, msg_correct, correct_list = correct_with_yaml(correct_filepath,
110-
path_to_scan, sheet_list)
111-
if not success:
112-
logger.info(f"No correction with yaml: {msg_correct}")
113-
else:
114-
sheet_list = correct_list
115-
logger.info("Success to correct with yaml.")
116-
success_to_write, writing_msg, result_file = write_output_file(output_file_without_ext,
117-
output_extension, sheet_list)
118-
if success_to_write:
119-
logger.info(f"Writing Output file({result_file}, success:{success_to_write}")
120-
else:
121-
logger.error(f"Fail to generate result file. msg:({writing_msg})")
122111
except Exception as ex:
123112
success = False
124113
msg = str(ex)

0 commit comments

Comments
 (0)