Skip to content

Commit 9dd6c94

Browse files
dd-jydd-jy
authored
Merge pull request #63 from fosslight/correct
Fix the correct mode bug for yaml format
2 parents 6156494 + 2352258 commit 9dd6c94

File tree

4 files changed

+55
-41
lines changed

4 files changed

+55
-41
lines changed

‎requirements.txt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
1-
fosslight_source
21
future
32
pandas
43
xlrd
54
openpyxl
65
progress
76
pyyaml
87
beautifulsoup4
9-
fosslight_util>=1.4.16
8+
fosslight_util>=1.4.25
9+
fosslight_source>=1.6.28
1010
fosslight_dependency>=3.7.4
11-
fosslight_binary>=4.0.7
11+
fosslight_binary>=4.1.24
1212
fosslight_prechecker>=3.0.1

‎src/fosslight_scanner/_help.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
-t\t\t\t Hide the progress bar
3535
-v\t\t\t Print FOSSLight Scanner version
3636
--no_correction\t Enter if you don't want to correct OSS information with sbom-info.yaml
37+
* Correction mode only supported xlsx format.
3738
--correct_fpath <path> Path to the sbom-info.yaml file
3839
3940
Options for only 'all' or 'bin' mode

‎src/fosslight_scanner/common.py‎

Lines changed: 25 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -201,15 +201,15 @@ def merge_yamls(_output_dir, merge_yaml_files, final_report, remove_src_data=Fal
201201
return success, err_msg
202202

203203

204-
def correct_scanner_result(_output_dir, output_files, exist_src, exist_bin):
204+
def correct_scanner_result(_output_dir, output_files, output_extension, exist_src, exist_bin):
205205
src_oss_list = []
206206
bin_oss_list = []
207207
duplicates = False
208208

209209
if exist_src:
210-
src_oss_list = check_exclude_dir(get_osslist_with_xlsx(_output_dir, output_files['SRC'], SRC_SHEET))
210+
src_oss_list = check_exclude_dir(get_osslist(_output_dir, output_files['SRC'], output_extension, SRC_SHEET))
211211
if exist_bin:
212-
bin_oss_list = check_exclude_dir(get_osslist_with_xlsx(_output_dir, output_files['BIN'], BIN_SHEET))
212+
bin_oss_list = check_exclude_dir(get_osslist(_output_dir, output_files['BIN'], output_extension, BIN_SHEET))
213213

214214
if exist_src and exist_bin:
215215
try:
@@ -250,11 +250,13 @@ def correct_scanner_result(_output_dir, output_files, exist_src, exist_bin):
250250

251251
try:
252252
if exist_src:
253-
success, err_msg = write_xlsx_with_osslist(src_oss_list, _output_dir, output_files['SRC'], SRC_SHEET)
253+
success, err_msg = write_output_with_osslist(src_oss_list, _output_dir, output_files['SRC'],
254+
output_extension, SRC_SHEET)
254255
if not success:
255256
logger.warning(err_msg)
256257
if exist_bin:
257-
success, err_msg = write_xlsx_with_osslist(bin_oss_list, _output_dir, output_files['BIN'], BIN_SHEET)
258+
success, err_msg = write_output_with_osslist(bin_oss_list, _output_dir, output_files['BIN'],
259+
output_extension, BIN_SHEET)
258260
if not success:
259261
logger.warning(err_msg)
260262
if duplicates:
@@ -264,35 +266,44 @@ def correct_scanner_result(_output_dir, output_files, exist_src, exist_bin):
264266
return
265267

266268

267-
def write_xlsx_with_osslist(oss_list, output_dir, output_file, sheetname):
269+
def write_output_with_osslist(oss_list, output_dir, output_file, output_extension, sheetname):
270+
new_oss_list = []
268271
sheet_list = {}
269272
sheet_list[sheetname] = []
270-
new_oss_list = []
271273

272274
for src_item in oss_list:
273275
new_oss_list.append(src_item.get_print_array()[0])
274276
sheet_list[sheetname].extend(new_oss_list)
275277
if os.path.exists(os.path.join(output_dir, output_file)):
276278
os.remove(os.path.join(output_dir, output_file))
277-
success, err_msg, result_file = write_output_file(os.path.join(output_dir, output_file).rstrip('xlsx'), '.xlsx',
278-
sheet_list)
279+
success, err_msg, _ = write_output_file(os.path.join(output_dir, output_file).rstrip(output_extension),
280+
output_extension, sheet_list)
279281
return success, err_msg
280282

281283

282-
def get_osslist_with_xlsx(_output_dir, output_file, sheet_name):
284+
def get_osslist(_output_dir, output_file, output_extension, sheet_name=''):
285+
err_reason = ''
283286
oss_list = []
284-
oss_xlsx = os.path.join(_output_dir, output_file)
285-
286-
if os.path.exists(oss_xlsx):
287-
oss_list.extend(read_oss_report(oss_xlsx, sheet_name))
287+
oss_file_with_fullpath = os.path.join(_output_dir, output_file)
288288

289+
if os.path.exists(oss_file_with_fullpath):
290+
if output_extension == '.xlsx':
291+
oss_list = read_oss_report(oss_file_with_fullpath, sheet_name)
292+
elif output_extension == '.yaml':
293+
oss_list, _, err_reason = parsing_yml(oss_file_with_fullpath, _output_dir)
294+
else:
295+
err_reason = f'Not supported extension: {output_extension}'
296+
if err_reason:
297+
logger.info(f'get_osslist: {err_reason}')
289298
return oss_list
290299

291300

292301
def check_exclude_dir(oss_list):
293302
_exclude_dirs = ["venv", "node_modules", "Pods", "Carthage"]
294303

295304
for oss_item in oss_list:
305+
if not oss_item.source_name_or_path:
306+
continue
296307
for exclude_dir in _exclude_dirs:
297308
if exclude_dir in oss_item.source_name_or_path[0].split(os.path.sep):
298309
oss_item.exclude = True

‎src/fosslight_scanner/fosslight_scanner.py‎

Lines changed: 26 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -148,11 +148,10 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
148148
-1, source_analysis,
149149
abs_path,
150150
src_output,
151-
False, num_cores, True)
151+
False, num_cores)
152152
if success:
153153
sheet_list["SRC_FL_Source"] = [scan_item.get_row_to_print() for scan_item in result[2]]
154-
need_license = True if output_extension == ".xlsx" else False
155-
create_report_file(0, result[2], result[3], 'all', need_license,
154+
create_report_file(0, result[2], result[3], 'all', False,
156155
_output_dir, output_files["SRC"].split('.')[0], output_extension,
157156
correct_mode, correct_fpath, abs_path)
158157
else: # Run fosslight_source by using docker image
@@ -192,36 +191,39 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
192191
try:
193192
output_file_without_ext = os.path.join(final_excel_dir, output_file)
194193
final_report = f"{output_file_without_ext}{output_extension}"
195-
tmp_dir = f"tmp_{datetime.now().strftime('%y%m%d_%H%M')}"
196-
exist_src = False
197-
exist_bin = False
198-
if correct_mode:
199-
os.makedirs(os.path.join(_output_dir, tmp_dir), exist_ok=True)
200-
if os.path.exists(os.path.join(_output_dir, output_files['SRC'])):
201-
exist_src = True
202-
shutil.copy2(os.path.join(_output_dir, output_files['SRC']), os.path.join(_output_dir, tmp_dir))
203-
if os.path.exists(os.path.join(_output_dir, output_files['BIN'])):
204-
exist_bin = True
205-
shutil.copy2(os.path.join(_output_dir, output_files['BIN']), os.path.join(_output_dir, tmp_dir))
206-
if exist_src or exist_bin:
207-
correct_scanner_result(_output_dir, output_files, exist_src, exist_bin)
194+
208195
if output_extension == ".xlsx":
196+
tmp_dir = f"tmp_{datetime.now().strftime('%y%m%d_%H%M')}"
197+
exist_src = False
198+
exist_bin = False
199+
if correct_mode:
200+
os.makedirs(os.path.join(_output_dir, tmp_dir), exist_ok=True)
201+
if os.path.exists(os.path.join(_output_dir, output_files['SRC'])):
202+
exist_src = True
203+
shutil.copy2(os.path.join(_output_dir, output_files['SRC']), os.path.join(_output_dir, tmp_dir))
204+
if os.path.exists(os.path.join(_output_dir, output_files['BIN'])):
205+
exist_bin = True
206+
shutil.copy2(os.path.join(_output_dir, output_files['BIN']), os.path.join(_output_dir, tmp_dir))
207+
if exist_src or exist_bin:
208+
correct_scanner_result(_output_dir, output_files, output_extension, exist_src, exist_bin)
209+
209210
if remove_src_data:
210211
overwrite_excel(_output_dir, default_oss_name, "OSS Name")
211212
overwrite_excel(_output_dir, url, "Download Location")
212213
success, err_msg = merge_excels(_output_dir, final_report)
214+
215+
if correct_mode:
216+
if exist_src:
217+
shutil.move(os.path.join(_output_dir, tmp_dir, output_files['SRC']),
218+
os.path.join(_output_dir, output_files['SRC']))
219+
if exist_bin:
220+
shutil.move(os.path.join(_output_dir, tmp_dir, output_files['BIN']),
221+
os.path.join(_output_dir, output_files['BIN']))
222+
shutil.rmtree(os.path.join(_output_dir, tmp_dir), ignore_errors=True)
213223
elif output_extension == ".yaml":
214224
merge_yaml_files = [output_files["SRC"], output_files["BIN"], output_files["DEP"]]
215225
success, err_msg = merge_yamls(_output_dir, merge_yaml_files, final_report,
216226
remove_src_data, default_oss_name, url)
217-
if correct_mode:
218-
if exist_src:
219-
shutil.move(os.path.join(_output_dir, tmp_dir, output_files['SRC']),
220-
os.path.join(_output_dir, output_files['SRC']))
221-
if exist_bin:
222-
shutil.move(os.path.join(_output_dir, tmp_dir, output_files['BIN']),
223-
os.path.join(_output_dir, output_files['BIN']))
224-
shutil.rmtree(os.path.join(_output_dir, tmp_dir), ignore_errors=True)
225227
if success:
226228
if os.path.isfile(final_report):
227229
result_log["Output File"] = final_report

0 commit comments

Comments
 (0)