Merge pull request #53 from fosslight/develop

Refactoring the code

Author: dd-jy

.reuse/dep5

@@ -85,3 +85,7 @@ License: Apache-2.0
 Files: .gitattributes 
 Copyright: 2021 LG Electronics
 License: Apache-2.0
+
+Files: tests/test_multi_pypi_npm/*
+Copyright: 2021 LG Electronics
+License: Apache-2.0

cli.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2020 LG Electronics Inc.
 # SPDX-License-Identifier: Apache-2.0
-from fosslight_dependency.analyze_dependency import main
+from fosslight_dependency.run_dependency_scanner import main
 
 
 if __name__ == '__main__':

setup.py

@@ -36,7 +36,7 @@
         include_package_data=True,
         entry_points={
             "console_scripts": [
-                "fosslight_dependency = fosslight_dependency.analyze_dependency:main"
+                "fosslight_dependency = fosslight_dependency.run_dependency_scanner:main"
             ]
         }
     )

src/fosslight_dependency/_analyze_dependency.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2021 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import logging
+import fosslight_dependency.constant as const
+from fosslight_dependency.package_manager.Pypi import Pypi
+from fosslight_dependency.package_manager.Npm import Npm
+from fosslight_dependency.package_manager.Maven import Maven
+from fosslight_dependency.package_manager.Gradle import Gradle
+from fosslight_dependency.package_manager.Pub import Pub
+from fosslight_dependency.package_manager.Cocoapods import Cocoapods
+from fosslight_dependency.package_manager.Android import Android
+from fosslight_dependency.package_manager.Swift import Swift
+from fosslight_dependency.package_manager.Carthage import Carthage
+import fosslight_util.constant as constant
+
+logger = logging.getLogger(constant.LOGGER_NAME)
+
+
+def analyze_dependency(package_manager_name, input_dir, output_dir, pip_activate_cmd='', pip_deactivate_cmd='',
+                       output_custom_dir='', app_name=const.default_app_name, github_token=''):
+    ret = True
+    package_sheet_list = []
+
+    if package_manager_name == const.PYPI:
+        package_manager = Pypi(input_dir, output_dir, pip_activate_cmd, pip_deactivate_cmd)
+    elif package_manager_name == const.NPM:
+        package_manager = Npm(input_dir, output_dir)
+    elif package_manager_name == const.MAVEN:
+        package_manager = Maven(input_dir, output_dir, output_custom_dir)
+    elif package_manager_name == const.GRADLE:
+        package_manager = Gradle(input_dir, output_dir, output_custom_dir)
+    elif package_manager_name == const.PUB:
+        package_manager = Pub(input_dir, output_dir)
+    elif package_manager_name == const.COCOAPODS:
+        package_manager = Cocoapods(input_dir, output_dir)
+    elif package_manager_name == const.ANDROID:
+        package_manager = Android(input_dir, output_dir, app_name)
+    elif package_manager_name == const.SWIFT:
+        package_manager = Swift(input_dir, output_dir, github_token)
+    elif package_manager_name == const.CARTHAGE:
+        package_manager = Carthage(input_dir, output_dir, github_token)
+    else:
+        logger.error("Not supported package manager name: " + package_manager_name)
+        ret = False
+        return ret, package_sheet_list
+
+    ret = package_manager.run_plugin()
+    if ret:
+        for f_name in package_manager.input_package_list_file:
+            logger.info("Parse oss information with file: " + str(f_name))
+
+            if os.path.isfile(f_name):
+                package_sheet_list = package_manager.parse_oss_information(f_name)
+            else:
+                logger.error('Failed to open input file:' + str(f_name))
+                ret = False
+
+    if ret:
+        logger.warning("### Complete to analyze: " + package_manager_name)
+    else:
+        logger.error("### Fail to analyze: " + package_manager_name)
+
+    return ret, package_sheet_list

src/fosslight_dependency/_package_manager.py

@@ -0,0 +1,178 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2021 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import sys
+import logging
+import platform
+import re
+import base64
+import fosslight_util.constant as constant
+import fosslight_dependency.constant as const
+
+try:
+    from github import Github
+except Exception:
+    pass
+
+logger = logging.getLogger(constant.LOGGER_NAME)
+
+# binary url to check license text
+_license_scanner_linux = "third_party/nomos/nomossa"
+_license_scanner_macos = "third_party/askalono/askalono_macos"
+_license_scanner_windows = "third_party\\askalono\\askalono.exe"
+
+
+class PackageManager:
+    input_package_list_file = []
+
+    def __init__(self, package_manager_name, dn_url, input_dir, output_dir):
+        self.input_package_list_file = []
+        self.package_manager_name = package_manager_name
+        self.input_dir = input_dir
+        self.output_dir = output_dir
+        self.dn_url = dn_url
+
+        self.platform = platform.system()
+        self.license_scanner_bin = check_license_scanner(self.platform)
+
+    def run_plugin(self):
+        logger.info('This package manager(' + self.package_manager_name + ') skips the step to run plugin.')
+        return True
+
+    def append_input_package_list_file(self, input_package_file):
+        self.input_package_list_file.append(input_package_file)
+
+
+def version_refine(oss_version):
+    version_cmp = oss_version.upper()
+
+    if version_cmp.find(".RELEASE") != -1:
+        oss_version = version_cmp.rstrip(".RELEASE")
+    elif version_cmp.find(".FINAL") != -1:
+        oss_version = version_cmp.rstrip(".FINAL")
+
+    return oss_version
+
+
+def connect_github(github_token):
+    if github_token is not None:
+        g = Github(github_token)
+    else:
+        g = Github()
+
+    return g
+
+
+def get_github_license(g, github_repo, platform, license_scanner_bin):
+    license_name = ''
+    tmp_license_txt_file_name = 'tmp_license.txt'
+
+    try:
+        repository = g.get_repo(github_repo)
+    except Exception:
+        logger.error("It cannot find the license name. Please use '-t' option with github token.")
+        logger.error("{0}{1}".format("refer:https://docs.github.com/en/github/authenticating-to-github/",
+                     "keeping-your-account-and-data-secure/creating-a-personal-access-token"))
+        repository = ''
+
+    if repository is not None:
+        try:
+            license_name = repository.get_license().license.spdx_id
+        except Exception:
+            logger.info("Cannot find the license name with github api.")
+
+        if license_name == "" or license_name == "NOASSERTION":
+            try:
+                license_txt_data = base64.b64decode(repository.get_license().content).decode('utf-8')
+                tmp_license_txt = open(tmp_license_txt_file_name, 'w', encoding='utf-8')
+                tmp_license_txt.write(license_txt_data)
+                tmp_license_txt.close()
+                license_name = check_and_run_license_scanner(platform, license_scanner_bin, tmp_license_txt_file_name)
+            except Exception:
+                logger.info("Cannot find the license name with license scanner binary.")
+
+            if os.path.isfile(tmp_license_txt_file_name):
+                os.remove(tmp_license_txt_file_name)
+
+    return license_name
+
+
+def check_license_scanner(platform):
+    license_scanner_bin = ''
+
+    if platform == const.LINUX:
+        license_scanner = _license_scanner_linux
+    elif platform == const.MACOS:
+        license_scanner = _license_scanner_macos
+    elif platform == const.WINDOWS:
+        license_scanner = _license_scanner_windows
+    else:
+        logger.debug("Not supported OS to analyze license text with binary.")
+
+    if license_scanner:
+        try:
+            base_path = sys._MEIPASS
+        except Exception:
+            base_path = os.path.dirname(__file__)
+
+        data_path = os.path.join(base_path, license_scanner)
+        license_scanner_bin = data_path
+
+    return license_scanner_bin
+
+
+def check_and_run_license_scanner(platform, license_scanner_bin, file_dir):
+    license_name = ''
+
+    if not license_scanner_bin:
+        logger.error('Not supported OS for license scanner binary.')
+
+    try:
+        tmp_output_file_name = "tmp_license_scanner_output.txt"
+
+        if file_dir == "UNKNOWN":
+            license_name = ""
+        else:
+            if platform == const.LINUX:
+                run_license_scanner = license_scanner_bin + " " + file_dir + " > " + tmp_output_file_name
+            elif platform == const.MACOS:
+                run_license_scanner = license_scanner_bin + " identify " + file_dir + " > " + tmp_output_file_name
+            elif platform == const.WINDOWS:
+                run_license_scanner = license_scanner_bin + " identify " + file_dir + " > " + tmp_output_file_name
+            else:
+                run_license_scanner = ''
+
+            if run_license_scanner is None:
+                license_name = ""
+                return license_name
+            else:
+                ret = os.system(run_license_scanner)
+                if ret != 0:
+                    logger.info("Cannot find the license name with scanner bin.")
+                    return ""
+
+            fp = open(tmp_output_file_name, "r", encoding='utf8')
+            license_output = fp.read()
+            fp.close()
+            os.remove(tmp_output_file_name)
+
+            if platform == const.LINUX:
+                license_output_re = re.findall(r'.*contains license\(s\)\s(.*)', license_output)
+            else:
+                license_output_re = re.findall(r"License:\s{1}(\S*)\s{1}", license_output)
+
+            if len(license_output_re) == 1:
+                license_name = license_output_re[0]
+                if license_name == "No_license_found":
+                    license_name = ""
+            else:
+                license_name = ""
+
+    except Exception as ex:
+        logger.error("Failed to run license scan binary." + str(ex))
+        license_name = ""
+
+    return license_name