Merge branch 'main' into develop

Author: dd-jy

.bumpversion.cfg

@@ -2,7 +2,7 @@
 commit = True
 tag = False
 message = Bump version: {current_version} → {new_version}
-current_version = 1.6.9
+current_version = 1.6.10
 
 [bumpversion:file:setup.py]
 search = '{current_version}'

CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## v1.6.10 (02/06/2022)
+## Changes
+- Amend how to handle path without files to scan for SCANOSS @JustinWonjaePark (#73)
+- Refine license name on SCANOSS results @JustinWonjaePark (#74)
+
+## 🚀 Features
+
+- Print SCANOSS info in a separate sheet @JustinWonjaePark (#72)
+- Add Dockerfile to build on ubuntu @soimkim (#69)
+
+## 🐛 Hotfixes
+
+- Fix cli.py to remove redundant report @JustinWonjaePark (#71)
+
+---
+
 ## v1.6.9 (06/04/2022)
 ## Changes
 ## 🐛 Hotfixes

Dockerfile

@@ -0,0 +1,25 @@
+# Copyright (c) 2021 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+FROM	ubuntu:20.04
+
+RUN 	apt-get update && apt-get install sudo -y
+RUN	ln -sf /bin/bash /bin/sh
+
+COPY	. /app
+WORKDIR	/app
+
+ENV 	DEBIAN_FRONTEND=noninteractive
+
+RUN	apt-get -y install build-essential
+RUN     apt-get -y install python3 python3-distutils python3-pip python3-dev
+RUN	apt-get -y install python3-intbitset python3-magic
+RUN	apt-get -y install libxml2-dev
+RUN	apt-get -y install libxslt1-dev
+RUN 	apt-get -y install libhdf5-dev
+RUN 	apt-get -y install bzip2 xz-utils zlib1g libpopt0 
+RUN	apt-get -y install gcc-10 g++-10
+RUN	pip3 install --upgrade pip
+RUN	pip3 install .
+RUN	pip3 install dparse
+
+ENTRYPOINT ["/usr/local/bin/fosslight_source"] 

requirements.txt

@@ -1,7 +1,7 @@
 pyparsing<=3.0.4;python_full_version<"3.6.8"
 scancode-toolkit
-typecode_libmagic
 XlsxWriter
+typecode-libmagic-from-sources
 fosslight_util>=1.3.21
 PyYAML
 wheel

setup.py

@@ -14,7 +14,7 @@
 if __name__ == "__main__":
     setup(
         name='fosslight_source',
-        version='1.6.9',
+        version='1.6.10',
         package_dir={"": "src"},
         packages=find_packages(where='src'),
         description='FOSSLight Source Scanner',

src/fosslight_source/_help.py

@@ -21,7 +21,7 @@
             -h\t\t\t\t   Print help message
             -v\t\t\t\t   Print FOSSLight Source Scanner version
             -j\t\t\t\t   Generate raw result of scanners in json format
-            -m\t\t\t\t   Print the Matched text for each license on a separate sheet (Scancode Only)
+            -m\t\t\t\t   Print additional information for scan result on separate sheets
             -o <output_path>\t\t   Output path
             \t\t\t\t    (If you want to generate the specific file name, add the output path with file name.)
             -f <format>\t\t\t   Output file format (excel, csv, opossum, yaml)

src/fosslight_source/_parsing_scancode_file_item.py

@@ -74,12 +74,12 @@ def parsing_file_item(scancode_file_list, has_error, need_matched_license=False)
                     error_msg = file["scan_errors"]
                     if len(error_msg) > 0:
                         logger.debug(f"Test_msg {file_path}:{error_msg}")
-                        result_item.set_comment(",".join(error_msg))
+                        result_item.comment = ",".join(error_msg)
                         scancode_file_item.append(result_item)
                         continue
 
                 copyright_value_list = [x["value"] for x in copyright_list]
-                result_item.set_copyright(copyright_value_list)
+                result_item.copyright = copyright_value_list
 
                 # Set the license value
                 license_detected = []
@@ -137,19 +137,18 @@ def parsing_file_item(scancode_file_list, has_error, need_matched_license=False)
 
                     matched_rule = lic_item["matched_rule"]
                     if matched_rule["is_license_text"]:
-                        result_item.set_is_license_text(True)
+                        result_item.is_license_text = True
 
                 if len(license_detected) > 0:
-                    result_item.set_licenses(license_detected)
+                    result_item.licenses = license_detected
 
                     if len(license_expression_list) > 0:
                         license_expression_list = list(
                             set(license_expression_list))
-                        result_item.set_comment(
-                            ','.join(license_expression_list))
+                        result_item.comment = ','.join(license_expression_list)
 
                     if is_exclude_file(file_path, prev_dir, prev_dir_value):
-                        result_item.set_exclude(True)
+                        result_item.exclude = True
                     scancode_file_item.append(result_item)
 
         except Exception as ex:

src/fosslight_source/_parsing_scanoss_file.py

@@ -7,8 +7,28 @@
 import fosslight_util.constant as constant
 from ._scan_item import ScanItem
 from ._scan_item import is_exclude_file
+from ._scan_item import replace_word
 
 logger = logging.getLogger(constant.LOGGER_NAME)
+SCANOSS_INFO_HEADER = ['No', 'Source Name or Path', 'Component Declared', 'SPDX Tag',
+                       'File Header', 'License File', 'Scancode',
+                       'scanoss_matched_lines', 'scanoss_fileURL']
+
+
+def parsing_extraInfo(scanned_result):
+    scanoss_extra_info = []
+    for scan_item in scanned_result:
+        license_w_source = scan_item.scanoss_reference
+        if license_w_source:
+            extra_item = [scan_item.file, ','.join(license_w_source['component_declared']),
+                          ','.join(license_w_source['file_spdx_tag']),
+                          ','.join(license_w_source['file_header']),
+                          ','.join(license_w_source['license_file']),
+                          ','.join(license_w_source['scancode']),
+                          scan_item.matched_lines, scan_item.fileURL]
+            scanoss_extra_info.append(extra_item)
+    scanoss_extra_info.insert(0, SCANOSS_INFO_HEADER)
+    return scanoss_extra_info
 
 
 def parsing_scanResult(scanoss_report):
@@ -21,39 +41,49 @@ def parsing_scanResult(scanoss_report):
                 continue
 
         if 'component' in findings[0]:
-            result_item.set_oss_name(findings[0]['component'])
+            result_item.oss_name = findings[0]['component']
         if 'version' in findings[0]:
-            result_item.set_oss_version(findings[0]['version'])
+            result_item.oss_version = findings[0]['version']
         if 'url' in findings[0]:
-            result_item.set_download_location(findings[0]['url'])
+            result_item.download_location = findings[0]['url']
 
         license_detected = []
+        license_w_source = {"component_declared": [], "file_spdx_tag": [],
+                            "file_header": [], "license_file": [], "scancode": []}
         copyright_detected = []
         if 'licenses' in findings[0]:
             for license in findings[0]['licenses']:
-                license_detected.append(license['name'].lower())
+
+                license_lower = license['name'].lower()
+                for word in replace_word:
+                    if word in license_lower:
+                        license_lower = license_lower.replace(word, "")
+                license_detected.append(license_lower)
+
+                if license['source'] not in license_w_source:
+                    license_w_source[license['source']] = []
+                license_w_source[license['source']].append(license['name'])
             if len(license_detected) > 0:
-                result_item.set_licenses(license_detected)
+                result_item.licenses = license_detected
+                result_item.scanoss_reference = license_w_source
         if 'copyrights' in findings[0]:
             for copyright in findings[0]['copyrights']:
                 copyright_detected.append(copyright['name'])
             if len(copyright_detected) > 0:
-                result_item.set_copyright(copyright_detected)
+                result_item.copyright = copyright_detected
 
         if is_exclude_file(file_path):
-            result_item.set_exclude(True)
+            result_item.exclude = True
 
-        if 'vendor' in findings[0]:
-            result_item.set_vendor(findings[0]['vendor'])
         if 'file_url' in findings[0]:
-            result_item.set_fileURL(findings[0]['file_url'])
+            result_item.fileURL = findings[0]['file_url']
         if 'matched' in findings[0]:
             if 'lines' in findings[0]:
-                result_item.set_matched_lines(f"{findings[0]['matched']} ({findings[0]['lines']})")
+                result_item.matched_lines = f"{findings[0]['matched']} ({findings[0]['lines']})"
             else:
-                result_item.set_matched_lines(f"{findings[0]['matched']}")
+                result_item.matched_lines = f"{findings[0]['matched']}"
         elif 'lines' in findings[0]:
-            result_item.set_matched_lines(f"({findings[0]['lines']})")
+            result_item.matched_lines = f"({findings[0]['lines']})"
 
         scanoss_file_item.append(result_item)
 

src/fosslight_source/_scan_item.py

@@ -22,71 +22,46 @@
 
 class ScanItem:
     file = ""
-    licenses = []
-    copyright = ""
+    scanoss_reference = {}
     exclude = False
     is_license_text = False
     oss_name = ""
     oss_version = ""
     download_location = ""
     matched_lines = ""
     fileURL = ""
-    vendor = ""
     license_reference = ""
 
     def __init__(self, value):
         self.file = value
-        self.copyright = []
-        self.licenses = []
+        self._copyright = []
+        self._licenses = []
         self.comment = ""
         self.exclude = False
         self.is_license_text = False
 
     def __del__(self):
         pass
 
-    def set_comment(self, value):
-        self.comment = value
+    @property
+    def copyright(self):
+        return self._copyright
 
-    def set_file(self, value):
-        self.file = value
-
-    def set_copyright(self, value):
-        self.copyright.extend(value)
-        if len(self.copyright) > 0:
-            self.copyright = list(set(self.copyright))
-
-    def set_licenses(self, value):
-        self.licenses.extend(value)
-        if len(self.licenses) > 0:
-            self.licenses = list(set(self.licenses))
-
-    def set_exclude(self, value):
-        self.exclude = value
-
-    def set_is_license_text(self, value):
-        self.is_license_text = value
+    @copyright.setter
+    def copyright(self, value):
+        self._copyright.extend(value)
+        if len(self._copyright) > 0:
+            self._copyright = list(set(self._copyright))
 
-    def set_oss_name(self, value):
-        self.oss_name = value
+    @property
+    def licenses(self):
+        return self._licenses
 
-    def set_oss_version(self, value):
-        self.oss_version = value
-
-    def set_download_location(self, value):
-        self.download_location = value
-
-    def set_matched_lines(self, value):
-        self.matched_lines = value
-
-    def set_fileURL(self, value):
-        self.fileURL = value
-
-    def set_vendor(self, value):
-        self.vendor = value
-
-    def set_license_reference(self, value):
-        self.license_reference = value
+    @licenses.setter
+    def licenses(self, value):
+        self._licenses.extend(value)
+        if len(self._licenses) > 0:
+            self._licenses = list(set(self._licenses))
 
     def get_row_to_print(self):
         print_rows = [self.file, self.oss_name, self.oss_version, ','.join(self.licenses), self.download_location, "",
@@ -98,15 +73,13 @@ def get_row_to_print(self):
     def get_row_to_print_for_scanoss(self):
         print_rows = [self.file, self.oss_name, self.oss_version, ','.join(self.licenses), self.download_location, "",
                       ','.join(self.copyright),
-                      "Exclude" if self.exclude else "",
-                      self.comment, self.matched_lines, self.fileURL, self.vendor]
+                      "Exclude" if self.exclude else "", self.comment]
         return print_rows
 
     def get_row_to_print_for_all_scanner(self):
         print_rows = [self.file, self.oss_name, self.oss_version, ','.join(self.licenses), self.download_location, "",
                       ','.join(self.copyright),
-                      "Exclude" if self.exclude else "",
-                      self.comment, self.license_reference, self.matched_lines, self.fileURL, self.vendor]
+                      "Exclude" if self.exclude else "", self.comment, self.license_reference]
         return print_rows
 
     def merge_scan_item(self, other):
@@ -136,8 +109,8 @@ def merge_scan_item(self, other):
             self.matched_lines = other.matched_lines
         if not self.fileURL:
             self.fileURL = other.fileURL
-        if not self.vendor:
-            self.vendor = other.vendor
+        if not self.scanoss_reference:
+            self.scanoss_reference = other.scanoss_reference
 
     def __eq__(self, other):
         return self.file == other.file