Skip to content

Commit 9603cdd

Browse files
soonhong99soonhong99
authored
Setting.json with source_scanner selection (#109)
Signed-off-by: soonhong99 <[email protected]>
1 parent e89e6c0 commit 9603cdd

File tree

5 files changed

+202
-104
lines changed

5 files changed

+202
-104
lines changed

‎src/fosslight_scanner/_parse_setting.py‎

Lines changed: 23 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ def parse_setting_json(data):
1111
dep_argument = data.get('dep_argument', '')
1212
output = data.get('output', '')
1313
format = data.get('format', '')
14-
link = data.get('link', "")
14+
link = data.get('link', '')
1515
db_url = data.get('db_url', '')
1616
timer = data.get('timer', False)
1717
raw = data.get('raw', False)
@@ -20,34 +20,49 @@ def parse_setting_json(data):
2020
correct_fpath = data.get('correct_fpath', '')
2121
ui = data.get('ui', False)
2222
exclude_path = data.get('exclude', [])
23-
23+
selected_source_scanner = data.get('selected_source_scanner', '')
24+
source_write_json_file = data.get('source_write_json_file', False)
25+
source_print_matched_text = data.get('source_print_matched_text', False)
26+
source_time_out = data.get('source_time_out', 120)
27+
binary_simple = data.get('binary_simple', False)
2428
str_lists = [mode, path, exclude_path]
25-
strings = [dep_argument, output, format, db_url, correct_fpath, link]
26-
booleans = [timer, raw, no_correction, ui]
29+
strings = [
30+
dep_argument, output, format, db_url,
31+
correct_fpath, link, selected_source_scanner
32+
]
33+
booleans = [timer, raw, no_correction, ui, source_write_json_file, source_print_matched_text, binary_simple]
34+
2735
is_incorrect = False
2836

2937
# check if json file is incorrect format
3038
for i, target in enumerate(str_lists):
31-
if not (isinstance(target, list) and all(isinstance(item, str) for item in target)):
39+
if not (isinstance(target, list) and
40+
all(isinstance(item, str) for item in target)):
3241
is_incorrect = True
3342
str_lists[i] = []
3443

3544
for i, target in enumerate(strings):
3645
if not isinstance(target, str):
3746
is_incorrect = True
38-
str_lists[i] = ''
47+
strings[i] = ''
3948

4049
for i, target in enumerate(booleans):
4150
if not isinstance(target, bool):
4251
is_incorrect = True
43-
str_lists[i] = False
52+
booleans[i] = False
4453

4554
if not isinstance(core, int):
4655
is_incorrect = True
4756
core = -1
4857

58+
if not isinstance(source_time_out, int):
59+
is_incorrect = True
60+
source_time_out = 120
61+
4962
if is_incorrect:
5063
print('Ignoring some values with incorrect format in the setting file.')
5164

5265
return mode, path, dep_argument, output, format, link, db_url, timer, \
53-
raw, core, no_correction, correct_fpath, ui, exclude_path
66+
raw, core, no_correction, correct_fpath, ui, exclude_path, \
67+
selected_source_scanner, source_write_json_file, source_print_matched_text, source_time_out, \
68+
binary_simple

‎src/fosslight_scanner/cli.py‎

Lines changed: 82 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -4,68 +4,107 @@
44
# SPDX-License-Identifier: Apache-2.0
55
import sys
66
import json
7+
import os
8+
import os.path
79
from argparse import ArgumentParser
10+
811
from ._help import print_help_msg
912
from .fosslight_scanner import run_main, PKG_NAME
1013
from ._parse_setting import parse_setting_json
1114
from fosslight_util.help import print_package_version
12-
import os.path
1315

1416

1517
def set_args(mode, path, dep_argument, output, format, link, db_url, timer,
1618
raw, core, no_correction, correct_fpath, ui, setting, exclude_path):
19+
20+
selected_source_scanner = "all"
21+
source_write_json_file = False
22+
source_print_matched_text = False
23+
source_time_out = 120
24+
binary_simple = False
25+
1726
if setting and os.path.isfile(setting):
1827
try:
1928
with open(setting, 'r', encoding='utf-8') as file:
2029
data = json.load(file)
2130
s_mode, s_path, s_dep_argument, s_output, s_format, s_link, s_db_url, s_timer, s_raw, s_core, \
22-
s_no_correction, s_correct_fpath, s_ui, s_exclude_path = parse_setting_json(data)
31+
s_no_correction, s_correct_fpath, s_ui, s_exclude_path, \
32+
s_selected_source_scanner, s_source_write_json_file, s_source_print_matched_text, \
33+
s_source_time_out, s_binary_simple = parse_setting_json(data)
2334

2435
# direct cli arguments have higher priority than setting file
25-
mode = mode if mode else s_mode
26-
path = path if path else s_path
27-
dep_argument = dep_argument if dep_argument else s_dep_argument
28-
output = output if output else s_output
29-
format = format if format else s_format
30-
link = link if link else s_link
31-
db_url = db_url if db_url else s_db_url
32-
timer = timer if timer else s_timer
33-
raw = raw if raw else s_raw
34-
core = core if core else s_core
35-
no_correction = no_correction if no_correction else s_no_correction
36-
correct_fpath = correct_fpath if correct_fpath else s_correct_fpath
37-
ui = ui if ui else s_ui
38-
exclude_path = exclude_path if exclude_path else s_exclude_path
36+
mode = mode or s_mode
37+
path = path or s_path
38+
dep_argument = dep_argument or s_dep_argument
39+
output = output or s_output
40+
format = format or s_format
41+
link = link or s_link
42+
db_url = db_url or s_db_url
43+
timer = timer or s_timer
44+
raw = raw or s_raw
45+
core = core if core != -1 else s_core
46+
no_correction = no_correction or s_no_correction
47+
correct_fpath = correct_fpath or s_correct_fpath
48+
ui = ui or s_ui
49+
exclude_path = exclude_path or s_exclude_path
50+
51+
# These options are only set from the setting file, not from CLI arguments
52+
selected_source_scanner = s_selected_source_scanner or selected_source_scanner
53+
source_write_json_file = s_source_write_json_file
54+
source_print_matched_text = s_source_print_matched_text
55+
source_time_out = s_source_time_out if s_source_time_out != 120 else source_time_out
56+
binary_simple = s_binary_simple
3957

4058
except Exception as e:
4159
print(f"Cannot open setting file: {e}")
4260
return mode, path, dep_argument, output, format, link, db_url, timer, \
43-
raw, core, no_correction, correct_fpath, ui, exclude_path
61+
raw, core, no_correction, correct_fpath, ui, exclude_path, \
62+
selected_source_scanner, source_write_json_file, source_print_matched_text, source_time_out, \
63+
binary_simple
4464

4565

4666
def main():
47-
parser = ArgumentParser(description='FOSSLight Scanner', prog='fosslight_scanner', add_help=False)
48-
parser.add_argument('mode', nargs='*', help='source| dependency| binary| all| compare', default="")
49-
parser.add_argument('--path', '-p', help='Path to analyze (In compare mode, two FOSSLight reports',
67+
parser = ArgumentParser(description='FOSSLight Scanner',
68+
prog='fosslight_scanner', add_help=False)
69+
parser.add_argument('mode', nargs='*',
70+
help='source| dependency| binary| all| compare',
71+
default="")
72+
parser.add_argument('--path', '-p',
73+
help='Path to analyze (In compare mode, two FOSSLight reports',
5074
dest='path', nargs='+', default="")
51-
parser.add_argument('--wget', '-w', help='Link to be analyzed', type=str, dest='link', default="")
52-
parser.add_argument('--format', '-f', help='Scanner output file format (excel,yaml), Compare mode (excel,html,yaml,json)',
75+
parser.add_argument('--wget', '-w', help='Link to be analyzed',
76+
type=str, dest='link', default="")
77+
parser.add_argument('--format', '-f',
78+
help='Scanner output file format (excel,yaml), Compare mode (excel,html,yaml,json)',
5379
type=str, dest='format', default="")
54-
parser.add_argument('--output', '-o', help='Output directory or file', type=str, dest='output', default="")
55-
parser.add_argument('--dependency', '-d', help='Dependency arguments', type=str, dest='dep_argument', default="")
56-
parser.add_argument('--url', '-u', help="DB Url", type=str, dest='db_url', default="")
57-
parser.add_argument('--core', '-c', help='Number of processes to analyze source', type=int, dest='core', default=-1)
58-
parser.add_argument('--raw', '-r', help='Keep raw data', action='store_true', dest='raw', default=False)
59-
parser.add_argument('--timer', '-t', help='Hide the progress bar', action='store_true', dest='timer', default=False)
60-
parser.add_argument('--version', '-v', help='Print version', action='store_true', dest='version', default=False)
61-
parser.add_argument('--help', '-h', help='Print help message', action='store_true', dest='help')
62-
parser.add_argument('--exclude', '-e', help='Path to exclude from analysis', dest='exclude_path', nargs='*', default=[])
63-
parser.add_argument('--setting', '-s', help='Scanner json setting file', type=str, dest='setting', default="")
64-
parser.add_argument('--no_correction', help='No correction with sbom-info.yaml',
80+
parser.add_argument('--output', '-o', help='Output directory or file',
81+
type=str, dest='output', default="")
82+
parser.add_argument('--dependency', '-d', help='Dependency arguments',
83+
type=str, dest='dep_argument', default="")
84+
parser.add_argument('--url', '-u', help="DB Url",
85+
type=str, dest='db_url', default="")
86+
parser.add_argument('--core', '-c',
87+
help='Number of processes to analyze source',
88+
type=int, dest='core', default=-1)
89+
parser.add_argument('--raw', '-r', help='Keep raw data',
90+
action='store_true', dest='raw', default=False)
91+
parser.add_argument('--timer', '-t', help='Hide the progress bar',
92+
action='store_true', dest='timer', default=False)
93+
parser.add_argument('--version', '-v', help='Print version',
94+
action='store_true', dest='version', default=False)
95+
parser.add_argument('--help', '-h', help='Print help message',
96+
action='store_true', dest='help')
97+
parser.add_argument('--exclude', '-e', help='Path to exclude from analysis',
98+
dest='exclude_path', nargs='*', default=[])
99+
parser.add_argument('--setting', '-s', help='Scanner json setting file',
100+
type=str, dest='setting', default="")
101+
parser.add_argument('--no_correction',
102+
help='No correction with sbom-info.yaml',
65103
action='store_true', required=False, default=False)
66104
parser.add_argument('--correct_fpath', help='Path to the sbom-info.yaml',
67105
type=str, required=False, default='')
68-
parser.add_argument('--ui', help='Generate UI mode result file', action='store_true', required=False, default=False)
106+
parser.add_argument('--ui', help='Generate UI mode result file',
107+
action='store_true', required=False, default=False)
69108

70109
try:
71110
args = parser.parse_args()
@@ -78,12 +117,17 @@ def main():
78117
print_package_version(PKG_NAME, "FOSSLight Scanner Version:")
79118
else:
80119
mode, path, dep_argument, output, format, link, db_url, timer, raw, core, no_correction, correct_fpath, \
81-
ui, exclude_path = set_args(args.mode, args.path, args.dep_argument, args.output, args.format,
82-
args.link, args.db_url, args.timer, args.raw, args.core, args.no_correction,
83-
args.correct_fpath, args.ui, args.setting, args.exclude_path)
120+
ui, exclude_path, selected_source_scanner, source_write_json_file, source_print_matched_text, \
121+
source_time_out, binary_simple, = set_args(
122+
args.mode, args.path, args.dep_argument, args.output,
123+
args.format, args.link, args.db_url, args.timer, args.raw,
124+
args.core, args.no_correction, args.correct_fpath, args.ui,
125+
args.setting, args.exclude_path)
84126

85127
run_main(mode, path, dep_argument, output, format, link, db_url, timer,
86-
raw, core, not no_correction, correct_fpath, ui, exclude_path)
128+
raw, core, not no_correction, correct_fpath, ui, exclude_path,
129+
selected_source_scanner, source_write_json_file, source_print_matched_text,
130+
source_time_out, binary_simple)
87131

88132

89133
if __name__ == "__main__":

‎src/fosslight_scanner/common.py‎

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import shutil
1010
import pandas as pd
1111
import yaml
12+
1213
import fosslight_util.constant as constant
1314
from fosslight_util.parsing_yaml import parsing_yml
1415
from fosslight_util.write_yaml import create_yaml_with_ossitem
@@ -20,11 +21,14 @@
2021
logger = logging.getLogger(constant.LOGGER_NAME)
2122
SRC_SHEET = 'SRC_FL_Source'
2223
BIN_SHEET = 'BIN_FL_Binary'
23-
BIN_EXT_HEADER = {'BIN_FL_Binary': ['ID', 'Binary Path', 'OSS Name',
24-
'OSS Version', 'License', 'Download Location',
25-
'Homepage', 'Copyright Text', 'Exclude',
26-
'Comment', 'Vulnerability Link', 'TLSH', 'SHA1']}
27-
BIN_HIDDEN_HEADER = {'TLSH', "SHA1"}
24+
BIN_EXT_HEADER = {
25+
'BIN_FL_Binary': [
26+
'ID', 'Binary Path', 'OSS Name', 'OSS Version', 'License',
27+
'Download Location', 'Homepage', 'Copyright Text', 'Exclude',
28+
'Comment', 'Vulnerability Link', 'TLSH', 'SHA1'
29+
]
30+
}
31+
BIN_HIDDEN_HEADER = {'TLSH', 'SHA1'}
2832

2933

3034
def copy_file(source, destination):
@@ -38,16 +42,15 @@ def copy_file(source, destination):
3842
except Exception as ex:
3943
logger.debug(f"Failed to copy {source} to {destination}: {ex}")
4044
return False, copied_file
41-
else:
42-
return True, copied_file
45+
return True, copied_file
4346

4447

4548
def run_analysis(path_to_run, params, func, str_run_start, output, exe_path):
4649
# This function will be replaced by call_analysis_api().
47-
logger.info("## Start to run "+str_run_start)
50+
logger.info("## Start to run " + str_run_start)
4851
return_value = ""
4952
try:
50-
if path_to_run != "":
53+
if path_to_run:
5154
logger.info(f"|--- Path to analyze : {path_to_run}")
5255
os.chdir(output)
5356
sys.argv = params
@@ -68,7 +71,7 @@ def call_analysis_api(path_to_run, str_run_start, return_idx, func, *args, **kwa
6871
success = True
6972
result = []
7073
try:
71-
if path_to_run != "":
74+
if path_to_run:
7275
logger.info(f"|--- Path to analyze : {path_to_run}")
7376
result = func(*args, **kwargs)
7477
else:
@@ -79,36 +82,33 @@ def call_analysis_api(path_to_run, str_run_start, return_idx, func, *args, **kwa
7982
success = False
8083
logger.error(f"{str_run_start}:{ex}")
8184
try:
82-
if success:
83-
if result and return_idx >= 0:
84-
if len(result) > return_idx:
85-
result = result[return_idx]
86-
else:
87-
success = False
85+
if success and result and return_idx >= 0:
86+
if len(result) > return_idx:
87+
result = result[return_idx]
88+
else:
89+
success = False
8890
except Exception as ex:
8991
logger.debug(f"Get return value:{ex}")
9092
success = False
91-
if not result:
92-
result = []
93-
return success, result
93+
return success, result or []
9494

9595

9696
def overwrite_excel(excel_file_path, oss_name, column_name='OSS Name'):
97-
if oss_name != "":
97+
if oss_name:
9898
try:
9999
files = os.listdir(excel_file_path)
100100
for file in files:
101101
if file.endswith(".xlsx"):
102-
file = os.path.join(excel_file_path, file)
103-
excel_file = pd.ExcelFile(file, engine='openpyxl')
102+
file_path = os.path.join(excel_file_path, file)
103+
excel_file = pd.ExcelFile(file_path, engine='openpyxl')
104104

105105
for sheet_name in excel_file.sheet_names:
106106
try:
107-
df = pd.read_excel(file, sheet_name=sheet_name, engine='openpyxl')
107+
df = pd.read_excel(file_path, sheet_name=sheet_name, engine='openpyxl')
108108
if column_name in df.columns:
109109
updated = (df[column_name] == '') | (df[column_name].isnull())
110110
df.loc[updated, column_name] = oss_name
111-
df.to_excel(file, sheet_name=sheet_name, index=False)
111+
df.to_excel(file_path, sheet_name=sheet_name, index=False)
112112
except Exception as ex:
113113
logger.debug(f"overwrite_sheet {sheet_name}:{ex}")
114114
except Exception as ex:

0 commit comments

Comments
 (0)