Change license scanner to askalono package (#242)

* Change license scanner to askalono package
* Remove third party src (nomos)

---------

Signed-off-by: jiyeong.seok <[email protected]>

Author: dd-jy

.gitattributes

@@ -1,3 +1,2 @@
 # Apply override to all files in the directory
 tests/** linguist-vendored
-src/fosslight_dependency/third_party/** linguist-vendored

.github/workflows/publish-release.yml

@@ -72,7 +72,7 @@ jobs:
         
     - name: Build exe with PyInstaller for windows
       run: |
-        pyinstaller --onefile cli.py -n cli --additional-hooks-dir=hooks --add-binary "src/fosslight_dependency/third_party/askalono/askalono.exe;third_party/askalono" --add-binary "LICENSE;LICENSES" --add-binary "LICENSES\LicenseRef-3rd_party_licenses.txt;LICENSES" --collect-datas fosslight_util --hidden-import=_cffi_backend
+        pyinstaller --onefile cli.py -n cli --additional-hooks-dir=hooks --add-binary "LICENSE;LICENSES" --add-binary "LICENSES\LicenseRef-3rd_party_licenses.txt;LICENSES" --collect-datas fosslight_util --hidden-import=_cffi_backend
         mkdir out
         move dist/cli.exe fosslight_dependency_windows.exe
                 

.reuse/dep5

@@ -22,18 +22,14 @@ Files: setup.cfg
 Copyright: 2021 LG Electronics
 License: Apache-2.0
 
-Files: .gitignore
+Files: requirements.txt
 Copyright: 2021 LG Electronics
-License: Apache-2.0
+License: LicenseRef-3rd_party_licenses
 
-Files: src/fosslight_dependency/third_party/askalono/*
-Copyright: 2018 Amazon.com, Inc. or its affiliates.
+Files: .gitignore
+Copyright: 2021 LG Electronics
 License: Apache-2.0
 
-Files: src/fosslight_dependency/third_party/nomos/*
-Copyright: 2006-2009 Hewlett-Packard Development Company, L.P.
-License: LicenseRef-3rd_party_licenses
-
 Files: tests/test_gradle/*
 Copyright: 2017-2021 Google LLC.
 License: Apache-2.0

setup.py

@@ -52,10 +52,7 @@
                      "Programming Language :: Python :: 3.8",
                      "Programming Language :: Python :: 3.9", ],
         install_requires=required,
-        package_data={_PACKAEG_NAME: [os.path.join('third_party', 'nomos', 'nomossa'),
-                                      os.path.join('third_party', 'askalono', 'askalono.exe'),
-                                      os.path.join('third_party', 'askalono', 'askalono_macos'),
-                                      os.path.join(_LICENSE_DIR, '*')]},
+        package_data={_PACKAEG_NAME: [os.path.join(_LICENSE_DIR, '*')]},
         include_package_data=True,
         entry_points={
             "console_scripts": [

src/fosslight_dependency/_package_manager.py

@@ -4,17 +4,17 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import os
-import sys
 import logging
 import platform
 import re
 import base64
 import subprocess
 import shutil
 import stat
+from packageurl.contrib import url2purl
+from askalono import identify
 import fosslight_util.constant as constant
 import fosslight_dependency.constant as const
-from packageurl.contrib import url2purl
 
 try:
     from github import Github
@@ -23,13 +23,9 @@
 
 logger = logging.getLogger(constant.LOGGER_NAME)
 
-# binary url to check license text
-_license_scanner_linux = os.path.join('third_party', 'nomos', 'nomossa')
-_license_scanner_macos = os.path.join('third_party', 'askalono', 'askalono_macos')
-_license_scanner_windows = os.path.join('third_party', 'askalono', 'askalono.exe')
-
 gradle_config = ['runtimeClasspath', 'runtime']
 android_config = ['releaseRuntimeClasspath']
+ASKALONO_THRESHOLD = 0.7
 
 
 class PackageManager:
@@ -54,7 +50,6 @@ def __init__(self, package_manager_name, dn_url, input_dir, output_dir):
         self.dep_items = []
 
         self.platform = platform.system()
-        self.license_scanner_bin = check_license_scanner(self.platform)
 
     def __del__(self):
         self.input_package_list_file = []
@@ -316,9 +311,8 @@ def connect_github(github_token):
     return g
 
 
-def get_github_license(g, github_repo, platform, license_scanner_bin):
+def get_github_license(g, github_repo):
     license_name = ''
-    tmp_license_txt_file_name = 'tmp_license.txt'
 
     try:
         repository = g.get_repo(github_repo)
@@ -334,96 +328,26 @@ def get_github_license(g, github_repo, platform, license_scanner_bin):
             if license_name == "" or license_name == "NOASSERTION":
                 try:
                     license_txt_data = base64.b64decode(repository.get_license().content).decode('utf-8')
-                    tmp_license_txt = open(tmp_license_txt_file_name, 'w', encoding='utf-8')
-                    tmp_license_txt.write(license_txt_data)
-                    tmp_license_txt.close()
-                    license_name = check_and_run_license_scanner(platform, license_scanner_bin, tmp_license_txt_file_name)
+                    license_name = check_license_name(license_txt_data)
                 except Exception:
-                    logger.info("Cannot find the license name with license scanner binary.")
-
-                if os.path.isfile(tmp_license_txt_file_name):
-                    os.remove(tmp_license_txt_file_name)
+                    logger.info("Cannot find the license name with askalono.")
         except Exception:
             logger.info("Cannot find the license name with github api.")
 
     return license_name
 
 
-def check_license_scanner(platform):
-    license_scanner_bin = ''
-
-    if platform == const.LINUX:
-        license_scanner = _license_scanner_linux
-    elif platform == const.MACOS:
-        license_scanner = _license_scanner_macos
-    elif platform == const.WINDOWS:
-        license_scanner = _license_scanner_windows
-    else:
-        logger.debug("Not supported OS to analyze license text with binary.")
-
-    if license_scanner:
-        try:
-            base_path = sys._MEIPASS
-        except Exception:
-            base_path = os.path.dirname(__file__)
-
-        data_path = os.path.join(base_path, license_scanner)
-        license_scanner_bin = data_path
-
-    return license_scanner_bin
-
-
-def check_and_run_license_scanner(platform, license_scanner_bin, file_dir):
+def check_license_name(license_txt, is_filepath=False):
     license_name = ''
+    if is_filepath:
+        with open(license_txt, 'r', encoding='utf-8') as f:
+            license_content = f.read()
+    else:
+        license_content = license_txt
 
-    if not license_scanner_bin:
-        logger.error('Not supported OS for license scanner binary.')
-
-    try:
-        tmp_output_file_name = "tmp_license_scanner_output.txt"
-
-        if file_dir == "UNKNOWN":
-            license_name = ""
-        else:
-            if platform == const.LINUX:
-                run_license_scanner = f"{license_scanner_bin} {file_dir} > {tmp_output_file_name}"
-            elif platform == const.MACOS:
-                run_license_scanner = f"{license_scanner_bin} identify {file_dir} > {tmp_output_file_name}"
-            elif platform == const.WINDOWS:
-                run_license_scanner = f"{license_scanner_bin} identify {file_dir} > {tmp_output_file_name}"
-            else:
-                run_license_scanner = ''
-
-            if run_license_scanner is None:
-                license_name = ""
-                return license_name
-            else:
-                ret = subprocess.run(run_license_scanner, shell=True, stderr=subprocess.PIPE)
-                if ret.returncode != 0 or ret.stderr:
-                    os.remove(tmp_output_file_name)
-                    return ""
-
-            fp = open(tmp_output_file_name, "r", encoding='utf8')
-            license_output = fp.read()
-            fp.close()
-
-            if platform == const.LINUX:
-                license_output_re = re.findall(r'.*contains license\(s\)\s(.*)', license_output)
-            else:
-                license_output_re = re.findall(r"License:\s{1}(\S*)\s{1}", license_output)
-
-            if len(license_output_re) == 1:
-                license_name = license_output_re[0]
-                if license_name == "No_license_found":
-                    license_name = ""
-            else:
-                license_name = ""
-            os.remove(tmp_output_file_name)
-
-    except Exception as ex:
-        logger.error(f"Failed to run license scan binary. {ex}")
-        license_name = ""
-
+    detect_askalono = identify(license_content)
+    if detect_askalono.score > ASKALONO_THRESHOLD:
+        license_name = detect_askalono.name
     return license_name
 
 

src/fosslight_dependency/package_manager/Carthage.py

@@ -9,8 +9,8 @@
 import fosslight_util.constant as constant
 import fosslight_dependency.constant as const
 from fosslight_dependency._package_manager import PackageManager
-from fosslight_dependency._package_manager import connect_github, get_github_license, check_and_run_license_scanner
-from fosslight_dependency._package_manager import get_url_to_purl
+from fosslight_dependency._package_manager import connect_github, get_github_license
+from fosslight_dependency._package_manager import get_url_to_purl, check_license_name
 from fosslight_dependency.dependency_item import DependencyItem
 from fosslight_util.oss_item import OssItem
 
@@ -79,17 +79,15 @@ def parse_oss_information(self, f_name):
                                 for license_file_reg in license_file_regs:
                                     match_result = re.match(license_file_reg, filename_in_dir.lower())
                                     if match_result is not None:
-                                        license_name = check_and_run_license_scanner(self.platform,
-                                                                                     self.license_scanner_bin,
-                                                                                     filename_with_checkout_path)
+                                        license_name = check_license_name(filename_with_checkout_path, True)
                                         find_license = True
                                         break
                     if license_name == '':
                         if repo == github:
                             try:
                                 if not g:
                                     g = connect_github(self.github_token)
-                                license_name = get_github_license(g, oss_path, self.platform, self.license_scanner_bin)
+                                license_name = get_github_license(g, oss_path)
                             except Exception as e:
                                 logger.warning(f"Failed to get license with github api: {e}")
                                 license_name == ''

src/fosslight_dependency/package_manager/Nuget.py

@@ -12,7 +12,7 @@
 import fosslight_util.constant as constant
 import fosslight_dependency.constant as const
 from fosslight_dependency._package_manager import PackageManager
-from fosslight_dependency._package_manager import check_and_run_license_scanner, get_url_to_purl
+from fosslight_dependency._package_manager import check_license_name, get_url_to_purl
 from fosslight_dependency.dependency_item import DependencyItem, change_dependson_to_purl
 from fosslight_util.oss_item import OssItem
 
@@ -73,14 +73,9 @@ def parse_oss_information(self, f_name):
                         if license_url is not None:
                             url_res = requests.get(license_url.text)
                             if url_res.status_code == 200:
-                                tmp_license_txt = open(tmp_license_txt_file_name, 'w', encoding='utf-8')
-                                tmp_license_txt.write(url_res.text)
-                                tmp_license_txt.close()
-                                license_name_with_license_scanner = check_and_run_license_scanner(self.platform,
-                                                                                                  self.license_scanner_bin,
-                                                                                                  tmp_license_txt_file_name)
-                                if license_name_with_license_scanner != "":
-                                    license_name = license_name_with_license_scanner
+                                license_name_with_scanner = check_license_name(url_res.text)
+                                if license_name_with_scanner != "":
+                                    license_name = license_name_with_scanner
                                 else:
                                     license_name = license_url.text
                     oss_item.license = license_name

src/fosslight_dependency/package_manager/Pub.py

@@ -10,11 +10,10 @@
 import shutil
 import yaml
 import subprocess
-from askalono import identify
 import fosslight_util.constant as constant
 import fosslight_dependency.constant as const
 from fosslight_dependency._package_manager import PackageManager
-from fosslight_dependency._package_manager import get_url_to_purl
+from fosslight_dependency._package_manager import get_url_to_purl, check_license_name
 from fosslight_dependency.dependency_item import DependencyItem, change_dependson_to_purl
 from fosslight_util.oss_item import OssItem
 
@@ -135,9 +134,7 @@ def parse_oss_information(self, f_name):
                 purl_dict[f'{oss_origin_name}({oss_item.version})'] = dep_item.purl
                 license_txt = json_data['license']
                 if license_txt is not None:
-                    detect_askalono = identify(license_txt)
-                    if detect_askalono.score > 0.7:
-                        oss_item.license = detect_askalono.name
+                    oss_item.license = check_license_name(license_txt)
 
                 if self.direct_dep:
                     if oss_origin_name not in self.total_dep_list:

src/fosslight_dependency/package_manager/Pypi.py

@@ -13,7 +13,7 @@
 import fosslight_util.constant as constant
 import fosslight_dependency.constant as const
 from fosslight_dependency._package_manager import PackageManager
-from fosslight_dependency._package_manager import check_and_run_license_scanner, get_url_to_purl
+from fosslight_dependency._package_manager import check_license_name, get_url_to_purl
 from fosslight_dependency.dependency_item import DependencyItem, change_dependson_to_purl
 from fosslight_util.oss_item import OssItem
 
@@ -302,12 +302,7 @@ def parse_oss_information(self, f_name):
                 if license_name is not None:
                     license_name = license_name.replace(';', ',')
                 else:
-                    license_file_dir = d['LicenseFile']
-                    license_name_with_lic_scanner = check_and_run_license_scanner(self.platform,
-                                                                                  self.license_scanner_bin,
-                                                                                  license_file_dir)
-                    if license_name_with_lic_scanner != "":
-                        license_name = license_name_with_lic_scanner
+                    license_name = check_license_name(d['LicenseFile'], True)
                 oss_item.license = license_name
 
                 if oss_init_name == self.package_name:

src/fosslight_dependency/package_manager/Swift.py

@@ -140,7 +140,7 @@ def parse_oss_information(self, f_name):
             github_repo = "/".join(oss_item.homepage.split('/')[-2:])
             dep_item.purl = get_url_to_purl(oss_item.download_location, self.package_manager_name, github_repo, oss_item.version)
             purl_dict[f'{oss_origin_name}({oss_item.version})'] = dep_item.purl
-            oss_item.license = get_github_license(g, github_repo, self.platform, self.license_scanner_bin)
+            oss_item.license = get_github_license(g, github_repo)
 
             if self.direct_dep and len(self.direct_dep_list) > 0:
                 if oss_origin_name in self.direct_dep_list: