Seperate common codes for scanning. Add scanoss.py as a scanner

Author: JustinWonjaePark

setup.py

@@ -31,10 +31,11 @@
                      "Programming Language :: Python :: 3.8",
                      "Programming Language :: Python :: 3.9", ],
         install_requires=required,
+        extras_require={":python_version>'3.6'": ["scanoss>=0.7.0"]},
         entry_points={
             "console_scripts": [
                 "fosslight_convert = fosslight_source.convert_scancode:main",
-                "fosslight_source = fosslight_source.run_scancode:main",
+                "fosslight_source = fosslight_source.cli:main",
                 "convert_scancode = fosslight_source.convert_scancode:main",
                 "run_scancode = fosslight_source.run_scancode:main"
             ]

src/fosslight_source/_help.py

@@ -19,11 +19,12 @@
 
         Optional
             -h\t\t\t\t   Print help message
-            -j\t\t\t\t   Generate additional result of executing ScanCode in json format
-            -m\t\t\t\t   Print the Matched text for each license on a separate sheet
+            -j\t\t\t\t   Generate raw result of scanners in json format
+            -m\t\t\t\t   Print the Matched text for each license on a separate sheet (Scancode Only)
             -o <output_path>\t\t   Output path
             \t\t\t\t    (If you want to generate the specific file name, add the output path with file name.)
-            -f <format>\t\t\t   Output file format (excel, csv, opossum)"""
+            -f <format>\t\t\t   Output file format (excel, csv, opossum)
+            -s <scanner>\t\t   Select which scanner to be run (scancode, scanoss, all)"""
 
 _HELP_MESSAGE_CONVERT = """
     Usage: fosslight_convert [option1] <arg1> [option2] <arg2>...

src/fosslight_source/_parsing_scancode_file_item.py

@@ -8,96 +8,18 @@
 import re
 import fosslight_util.constant as constant
 from ._license_matched import MatchedLicense
+from ._scan_item import ScanItem
+from ._scan_item import is_exclude_dir
+from ._scan_item import is_exclude_file
+from ._scan_item import replace_word
 
 logger = logging.getLogger(constant.LOGGER_NAME)
-_replace_word = ["-only", "-old-style", "-or-later", "licenseref-scancode-"]
-_exclude_filename = ["changelog", "config.guess", "config.sub",
-                     "config.h.in", "changes", "ltmain.sh",
-                     "aclocal.m4", "configure", "configure.ac",
-                     "depcomp", "compile", "missing", "libtool.m4",
-                     "makefile"]
 _exclude_directory = ["test", "tests", "doc", "docs"]
 _exclude_directory = [os.path.sep + dir_name +
                       os.path.sep for dir_name in _exclude_directory]
 _exclude_directory.append("/.")
 
 
-class ScanCodeItem:
-    file = ""
-    licenses = []
-    copyright = ""
-    exclude = False
-    is_license_text = False
-
-    def __init__(self, value):
-        self.file = value
-        self.copyright = []
-        self.licenses = []
-        self.comment = ""
-        self.exclude = False
-        self.is_license_text = False
-
-    def __del__(self):
-        pass
-
-    def set_comment(self, value):
-        self.comment = value
-
-    def set_file(self, value):
-        self.file = value
-
-    def set_copyright(self, value):
-        self.copyright.extend(value)
-        if len(self.copyright) > 0:
-            self.copyright = list(set(self.copyright))
-
-    def set_licenses(self, value):
-        self.licenses.extend(value)
-        if len(self.licenses) > 0:
-            self.licenses = list(set(self.licenses))
-
-    def set_exclude(self, value):
-        self.exclude = value
-
-    def set_is_license_text(self, value):
-        self.is_license_text = value
-
-    def get_row_to_print(self):
-        print_rows = [self.file, "", "", ','.join(self.licenses), "", "",
-                      ','.join(self.copyright),
-                      "Exclude" if self.exclude else "",
-                      self.comment]
-        return print_rows
-
-
-def is_exclude_dir(dir_path):
-    if dir_path != "":
-        dir_path = dir_path.lower()
-        dir_path = dir_path if dir_path.endswith(
-            os.path.sep) else dir_path + os.path.sep
-        dir_path = dir_path if dir_path.startswith(
-            os.path.sep) else os.path.sep + dir_path
-        return any(dir_name in dir_path for dir_name in _exclude_directory)
-    return False
-
-
-def is_exclude_file(file_path, prev_dir, prev_dir_exclude_value):
-    file_path = file_path.lower()
-    filename = os.path.basename(file_path)
-    if filename in _exclude_filename:
-        return True
-
-    dir_path = os.path.dirname(file_path)
-    if dir_path == prev_dir:
-        return prev_dir_exclude_value
-    else:
-        # There will be no execution of this else statement.
-        # Because scancode json output results are sorted by path,
-        # most of them will match the previous if statement.
-        return is_exclude_dir(dir_path)
-    return False
-
-
 def get_error_from_header(header_item):
     has_error = False
     str_error = ""
@@ -146,7 +68,7 @@ def parsing_file_item(scancode_file_list, has_error, need_matched_license=False)
                 licenses = file["licenses"]
                 copyright_list = file["copyrights"]
 
-                result_item = ScanCodeItem(file_path)
+                result_item = ScanItem(file_path)
 
                 if has_error and "scan_errors" in file:
                     error_msg = file["scan_errors"]
@@ -196,7 +118,7 @@ def parsing_file_item(scancode_file_list, has_error, need_matched_license=False)
                             except Exception:
                                 pass
 
-                        for word in _replace_word:
+                        for word in replace_word:
                             if word in license_value:
                                 license_value = license_value.replace(word, "")
                         license_detected.append(license_value)

src/fosslight_source/_parsing_scanoss_file.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+import fosslight_util.constant as constant
+from ._scan_item import ScanItem
+from ._scan_item import is_exclude_file
+
+logger = logging.getLogger(constant.LOGGER_NAME)
+
+
+def parsing_scanResult(scanoss_report):
+    scanoss_file_item = []
+
+    for file_path, findings in scanoss_report.items():
+        result_item = ScanItem(file_path)
+        if 'id' in findings[0]:
+            if "none" == findings[0]['id']:
+                continue
+
+        if 'component' in findings[0]:
+            result_item.set_oss_name(findings[0]['component'])
+        if 'version' in findings[0]:
+            result_item.set_oss_version(findings[0]['version'])
+        if 'url' in findings[0]:
+            result_item.set_download_location(findings[0]['url'])
+
+        license_detected = []
+        copyright_detected = []
+        if 'licenses' in findings[0]:
+            for license in findings[0]['licenses']:
+                license_detected.append(license['name'].lower())
+            if len(license_detected) > 0:
+                result_item.set_licenses(license_detected)
+        if 'copyrights' in findings[0]:
+            for copyright in findings[0]['copyrights']:
+                copyright_detected.append(copyright['name'])
+            if len(copyright_detected) > 0:
+                result_item.set_copyright(copyright_detected)
+
+        if is_exclude_file(file_path):
+            result_item.set_exclude(True)
+
+        comment = ""
+        comment += "Match Type : " + findings[0]['id'] + ", "
+        if 'vendor' in findings[0]:
+            comment += "Vendor : " + findings[0]['vendor'] + ", "
+        if 'file_url' in findings[0]:
+            comment += "File URL : " + findings[0]['file_url'] + ", "
+        if 'matched' in findings[0]:
+            comment += "Matched : " + findings[0]['matched'] + ", "
+        if 'lines' in findings[0]:
+            comment += "Lines : " + findings[0]['lines']
+        result_item.set_comment(comment)
+
+        scanoss_file_item.append(result_item)
+
+    return scanoss_file_item

src/fosslight_source/_scan_item.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import logging
+import fosslight_util.constant as constant
+
+logger = logging.getLogger(constant.LOGGER_NAME)
+replace_word = ["-only", "-old-style", "-or-later", "licenseref-scancode-"]
+_exclude_filename = ["changelog", "config.guess", "config.sub",
+                     "config.h.in", "changes", "ltmain.sh",
+                     "aclocal.m4", "configure", "configure.ac",
+                     "depcomp", "compile", "missing", "libtool.m4",
+                     "makefile"]
+_exclude_directory = ["test", "tests", "doc", "docs"]
+_exclude_directory = [os.path.sep + dir_name +
+                      os.path.sep for dir_name in _exclude_directory]
+_exclude_directory.append("/.")
+
+
+class ScanItem:
+    file = ""
+    licenses = []
+    copyright = ""
+    exclude = False
+    is_license_text = False
+    oss_name = ""
+    oss_version = ""
+    download_location = ""
+
+    def __init__(self, value):
+        self.file = value
+        self.copyright = []
+        self.licenses = []
+        self.comment = ""
+        self.exclude = False
+        self.is_license_text = False
+
+    def __del__(self):
+        pass
+
+    def set_comment(self, value):
+        self.comment = value
+
+    def set_file(self, value):
+        self.file = value
+
+    def set_copyright(self, value):
+        self.copyright.extend(value)
+        if len(self.copyright) > 0:
+            self.copyright = list(set(self.copyright))
+
+    def set_licenses(self, value):
+        self.licenses.extend(value)
+        if len(self.licenses) > 0:
+            self.licenses = list(set(self.licenses))
+
+    def set_exclude(self, value):
+        self.exclude = value
+
+    def set_is_license_text(self, value):
+        self.is_license_text = value
+
+    def set_oss_name(self, value):
+        self.oss_name = value
+
+    def set_oss_version(self, value):
+        self.oss_version = value
+
+    def set_download_location(self, value):
+        self.download_location = value
+
+    def get_row_to_print(self):
+        if not self.download_location:
+            print_rows = [self.file, "", "", ','.join(self.licenses), "", "",
+                          ','.join(self.copyright),
+                          "Exclude" if self.exclude else "",
+                          self.comment]
+        else:
+            print_rows = [self.file, self.oss_name, self.oss_version, ','.join(self.licenses), self.download_location, "",
+                          ','.join(self.copyright),
+                          "Exclude" if self.exclude else "",
+                          self.comment]
+        return print_rows
+
+    def merge_scan_item(self, other):
+        """
+        Merge two ScanItem instance into one.
+
+        TODO: define how to merge comments and implement.
+        """
+        self.licenses = list(set(self.licenses + other.licenses))
+
+        if len(self.copyright) > 0:
+            self.copyright = list(set(self.copyright))
+
+        if self.exclude and other.exclude:
+            self.exclude = True
+        else:
+            self.exclude = False
+
+        if not self.oss_name:
+            self.oss_name = other.oss_name
+        if not self.oss_version:
+            self.oss_version = other.oss_version
+        if not self.download_location:
+            self.download_location = other.download_location
+
+    def __eq__(self, other):
+        return self.file == other.file
+
+
+def is_exclude_dir(dir_path):
+    if dir_path != "":
+        dir_path = dir_path.lower()
+        dir_path = dir_path if dir_path.endswith(
+            os.path.sep) else dir_path + os.path.sep
+        dir_path = dir_path if dir_path.startswith(
+            os.path.sep) else os.path.sep + dir_path
+        return any(dir_name in dir_path for dir_name in _exclude_directory)
+    return False
+
+
+def is_exclude_file(file_path, prev_dir=None, prev_dir_exclude_value=None):
+    file_path = file_path.lower()
+    filename = os.path.basename(file_path)
+    if filename in _exclude_filename:
+        return True
+
+    dir_path = os.path.dirname(file_path)
+    if prev_dir is not None:  # running ScanCode
+        if dir_path == prev_dir:
+            return prev_dir_exclude_value
+        else:
+            # There will be no execution of this else statement.
+            # Because scancode json output results are sorted by path,
+            # most of them will match the previous if statement.
+            return is_exclude_dir(dir_path)
+    else:  # running SCANOSS
+        return is_exclude_dir(dir_path)
+    return False