Merge and Apply Flake8 again

Author: bjk7119

setup.py

@@ -14,7 +14,7 @@
 if __name__ == "__main__":
     setup(
         name='fosslight_source',
-        version='1.4.7',
+        version='1.4.8',
         package_dir={"": "src"},
         packages=find_packages(where='src'),
         description='FOSSLight Source',

src/fosslight_source/_parsing_scancode_file_item.py

@@ -96,14 +96,34 @@ def is_exclude_file(file_path, prev_dir, prev_dir_exclude_value):
     return False
 
 
-def parsing_file_item(scancode_file_list):
+def get_error_from_header(header_item):
+    has_error = False
+    str_error = ""
+    key_error = "errors"
+
+    try:
+        for header in header_item:
+            if key_error in header:
+                errors = header[key_error]
+                error_cnt = len(errors)
+                if error_cnt > 0:
+                    has_error = True
+                    str_error = '{}...({})'.format(errors[0], error_cnt)
+                    break
+    except Exception as ex:
+        logger.debug("error_parsing_header:"+str(ex))
+    return has_error, str_error
+
+
+def parsing_file_item(scancode_file_list, has_error):
 
     rc = True
     scancode_file_item = []
     msg = "TOTAL FILE COUNT: " + str(len(scancode_file_list)) + "\n"
 
     prev_dir = ""
     prev_dir_value = False
+
     for file in scancode_file_list:
         try:
             is_binary = False
@@ -124,6 +144,14 @@ def parsing_file_item(scancode_file_list):
 
                 result_item = ScanCodeItem(file_path)
 
+                if has_error and "scan_errors" in file:
+                    error_msg = file["scan_errors"]
+                    if len(error_msg) > 0:
+                        logger.debug("test_msg" + file_path + ":" + str(error_msg))
+                        result_item.set_comment(",".join(error_msg))
+                        scancode_file_item.append(result_item)
+                        continue
+
                 copyright_value_list = [x["value"] for x in copyright_list]
                 result_item.set_copyright(copyright_value_list)
 
@@ -174,10 +202,11 @@ def parsing_file_item(scancode_file_list):
 
                     if is_exclude_file(file_path, prev_dir, prev_dir_value):
                         result_item.set_exclude(True)
-
                     scancode_file_item.append(result_item)
+
         except Exception as ex:
             msg += "* Error Parsing item:"+str(ex)
             rc = False
+            logger.debug(msg)
 
     return rc, scancode_file_item, msg.strip()

src/fosslight_source/convert_scancode.py

@@ -13,7 +13,7 @@
 from fosslight_util.set_log import init_log
 from fosslight_util.set_log import init_log_item
 import yaml
-from ._parsing_scancode_file_item import parsing_file_item
+from ._parsing_scancode_file_item import parsing_file_item, get_error_from_header
 from fosslight_util.write_excel import write_excel_and_csv
 from ._help import print_help_msg_convert
 
@@ -61,8 +61,8 @@ def convert_json_to_excel(scancode_json, excel_name):
         success = False
         logger.warning(str(ex))
 
-    scan_result_msg = str(success) + " " + msg
-    _result_log["Scan Result"] = scan_result_msg.strip()
+    scan_result_msg = str(success) if msg == "" else str(success) + "," + msg
+    _result_log["Scan Result"] = scan_result_msg
 
     try:
         _str_final_result_log = yaml.safe_dump(_result_log, allow_unicode=True, sort_keys=True)
@@ -79,8 +79,11 @@ def get_detected_licenses_from_scancode(scancode_json_file):
         logger.info("Start parsing " + scancode_json_file)
         with open(scancode_json_file, "r") as st_json:
             st_python = json.load(st_json)
-            rc, file_list, msg = parsing_file_item(st_python["files"])
-            logger.info("|---" + msg)
+            has_error, str_error = get_error_from_header(st_python["headers"])
+            rc, file_list, msg = parsing_file_item(st_python["files"], has_error)
+            logger.info("|---"+msg)
+            if has_error:
+                logger.info("|---Scan error:"+str_error)
     except Exception as error:
         logger.warning("Parsing " + scancode_json_file + ":" + str(error))
     logger.info("|---Number of files detected: " + str(len(file_list)))

src/fosslight_source/run_scancode.py

@@ -18,6 +18,7 @@
 from fosslight_util.set_log import init_log_item
 from fosslight_util.timer_thread import TimerThread
 from ._parsing_scancode_file_item import parsing_file_item
+from ._parsing_scancode_file_item import get_error_from_header
 from fosslight_util.write_excel import write_excel_and_csv
 from ._help import print_help_msg_source
 
@@ -97,25 +98,34 @@ def run_scan(path_to_scan, output_file_name="",
                                        processes=num_cores,
                                        output_json_pp=output_json_file,
                                        only_findings=True)
+
             if not rc:
-                msg += "Source code analysis failed."
+                msg = "Source code analysis failed."
                 success = False
+
             if results:
                 sheet_list = {}
-                for key, value in results.items():
-                    if key == "files":
-                        rc, result_list, parsing_msg = parsing_file_item(value)
-                        _result_log["Parsing Log"] = parsing_msg
-                        if rc:
-                            result_list = sorted(
-                                result_list, key=lambda row: (''.join(row.licenses)))
-                            sheet_list["SRC"] = [scan_item.get_row_to_print() for scan_item in result_list]
-
-                            success_to_write, writing_msg = write_excel_and_csv(
-                                output_file, sheet_list)
-                            logger.info("Writing excel :" + str(success_to_write) + " " + writing_msg)
-                            if success_to_write:
-                                _result_log["OSS Report"] = output_file + ".xlsx"
+                has_error = False
+                if "headers" in results:
+                    has_error, error_msg = get_error_from_header(results["headers"])
+                    if has_error:
+                        _result_log["Error_files"] = error_msg
+                        msg = "Failed to analyze :" + error_msg
+                if "files" in results:
+                    rc, result_list, parsing_msg = parsing_file_item(results["files"], has_error)
+                    _result_log["Parsing Log"] = parsing_msg
+                    if rc:
+                        if not success:
+                            success = True
+                        result_list = sorted(
+                            result_list, key=lambda row: (''.join(row.licenses)))
+                        sheet_list["SRC"] = [scan_item.get_row_to_print() for scan_item in result_list]
+
+                        success_to_write, writing_msg = write_excel_and_csv(
+                            output_file, sheet_list)
+                        logger.info("Writing excel :" + str(success_to_write) + " " + writing_msg)
+                        if success_to_write:
+                            _result_log["OSS Report"] = output_file + ".xlsx"
         except Exception as ex:
             success = False
             msg = str(ex)
@@ -126,8 +136,9 @@ def run_scan(path_to_scan, output_file_name="",
 
     if not return_results:
         result_list = []
-    scan_result_msg = str(success) + " " + msg
-    _result_log["Scan Result"] = scan_result_msg.strip()
+
+    scan_result_msg = str(success) if msg == "" else str(success) + "," + msg
+    _result_log["Scan Result"] = scan_result_msg
     _result_log["Output Directory"] = output_dir
     try:
         _str_final_result_log = yaml.safe_dump(_result_log, allow_unicode=True, sort_keys=True)