Merge pull request #73 from fosslight/develop

Modify to analyze the license name for carthage

Author: dd-jy

src/fosslight_dependency/_package_manager.py

@@ -77,30 +77,29 @@ def get_github_license(g, github_repo, platform, license_scanner_bin):
     try:
         repository = g.get_repo(github_repo)
     except Exception:
-        logger.error("It cannot find the license name. Please use '-t' option with github token.")
-        logger.error("{0}{1}".format("refer:https://docs.github.com/en/github/authenticating-to-github/",
-                     "keeping-your-account-and-data-secure/creating-a-personal-access-token"))
+        logger.info("It cannot find the license name. Please use '-t' option with github token.")
+        logger.info("{0}{1}".format("refer:https://docs.github.com/en/github/authenticating-to-github/",
+                    "keeping-your-account-and-data-secure/creating-a-personal-access-token"))
         repository = ''
 
-    if repository is not None:
+    if repository != '':
         try:
             license_name = repository.get_license().license.spdx_id
+            if license_name == "" or license_name == "NOASSERTION":
+                try:
+                    license_txt_data = base64.b64decode(repository.get_license().content).decode('utf-8')
+                    tmp_license_txt = open(tmp_license_txt_file_name, 'w', encoding='utf-8')
+                    tmp_license_txt.write(license_txt_data)
+                    tmp_license_txt.close()
+                    license_name = check_and_run_license_scanner(platform, license_scanner_bin, tmp_license_txt_file_name)
+                except Exception:
+                    logger.info("Cannot find the license name with license scanner binary.")
+
+                if os.path.isfile(tmp_license_txt_file_name):
+                    os.remove(tmp_license_txt_file_name)
         except Exception:
             logger.info("Cannot find the license name with github api.")
 
-        if license_name == "" or license_name == "NOASSERTION":
-            try:
-                license_txt_data = base64.b64decode(repository.get_license().content).decode('utf-8')
-                tmp_license_txt = open(tmp_license_txt_file_name, 'w', encoding='utf-8')
-                tmp_license_txt.write(license_txt_data)
-                tmp_license_txt.close()
-                license_name = check_and_run_license_scanner(platform, license_scanner_bin, tmp_license_txt_file_name)
-            except Exception:
-                logger.info("Cannot find the license name with license scanner binary.")
-
-            if os.path.isfile(tmp_license_txt_file_name):
-                os.remove(tmp_license_txt_file_name)
-
     return license_name
 
 

src/fosslight_dependency/package_manager/Carthage.py

@@ -5,15 +5,23 @@
 
 import logging
 import re
+import os
 import fosslight_util.constant as constant
 import fosslight_dependency.constant as const
 from fosslight_dependency._package_manager import PackageManager
 from fosslight_dependency._package_manager import connect_github
 from fosslight_dependency._package_manager import get_github_license
+from fosslight_dependency._package_manager import check_and_run_license_scanner
 
 logger = logging.getLogger(constant.LOGGER_NAME)
 
 
+checkout_dir = os.path.join("Carthage", 'Checkouts')
+license_file_regs = ['licen[cs]e[s]?', 'notice[s]?', 'copying*', 'copyright[s]?',
+                     'mit', 'bsd[-]?[0-4]?', 'bsd[-]?[0-4][-]?clause[s]?', 'apache[-,_]?[1-2]?[.,-,_]?[0-2]?',
+                     'unlicen[cs]e', '[a,l]?gpl[-]?[1-3]?[.,-,_]?[0-1]?', 'legal']
+
+
 class Carthage(PackageManager):
     package_manager_name = const.CARTHAGE
 
@@ -26,30 +34,75 @@ def __init__(self, input_dir, output_dir, github_token):
         self.append_input_package_list_file(self.input_file_name)
 
     def parse_oss_information(self, f_name):
+        github = "github"
+        checkout_dir_list = get_checkout_dirname()
         with open(f_name, 'r', encoding='utf8') as input_fp:
             sheet_list = []
-
-            g = connect_github(self.github_token)
+            g = ''
+            if not checkout_dir_list:
+                g = connect_github(self.github_token)
 
             for i, line in enumerate(input_fp.readlines()):
-
-                re_result = re.findall(r'github[\s]\"(\S*)\"[\s]\"(\S*)\"', line)
+                # Cartfile origin : github, git
+                # Ref. https://github.com/Carthage/Carthage/blob/master/Documentation/Artifacts.md
+                re_result = re.findall(r'(github|git)[\s]\"(\S*)\"[\s]\"(\S*)\"', line)
                 try:
-                    github_repo = re_result[0][0]
-                    oss_origin_name = github_repo.split('/')[1]
+                    repo = re_result[0][0]
+                    oss_path = re_result[0][1]
+                    if oss_path.endswith('.git'):
+                        oss_path = oss_path[:-4]
+                    oss_origin_name = oss_path.split('/')[-1]
                     oss_name = self.package_manager_name + ":" + oss_origin_name
-                    oss_version = re_result[0][1]
-                    homepage = self.dn_url + github_repo
+
+                    if repo == github:
+                        homepage = self.dn_url + oss_path
+                    else:
+                        homepage = oss_path
                     dn_loc = homepage
 
+                    oss_version = re_result[0][2]
+
                     license_name = ''
-                    license_name = get_github_license(g, github_repo, self.platform, self.license_scanner_bin)
+                    find_license = False
+                    if oss_origin_name in checkout_dir_list:
+                        oss_path_in_checkout = os.path.join(checkout_dir, oss_origin_name)
+                        for filename_in_dir in os.listdir(oss_path_in_checkout):
+                            if find_license:
+                                break
+                            filename_with_checkout_path = os.path.join(oss_path_in_checkout, filename_in_dir)
+                            if os.path.isfile(filename_with_checkout_path):
+                                for license_file_reg in license_file_regs:
+                                    match_result = re.match(license_file_reg, filename_in_dir.lower())
+                                    if match_result is not None:
+                                        license_name = check_and_run_license_scanner(self.platform,
+                                                                                     self.license_scanner_bin,
+                                                                                     filename_with_checkout_path)
+                                        find_license = True
+                                        break
+                    if license_name == '':
+                        if repo == github:
+                            try:
+                                if not g:
+                                    g = connect_github(self.github_token)
+                                license_name = get_github_license(g, oss_path, self.platform, self.license_scanner_bin)
+                            except Exception as e:
+                                logger.warning("Failed to get license with github api:" + str(e))
+                                license_name == ''
 
-                except Exception:
-                    logger.error("It cannot find the github oss information. So skip it.")
-                    break
+                    sheet_list.append([const.SUPPORT_PACKAE.get(self.package_manager_name),
+                                      oss_name, oss_version, license_name, dn_loc, homepage, '', '', ''])
 
-                sheet_list.append([const.SUPPORT_PACKAE.get(self.package_manager_name),
-                                  oss_name, oss_version, license_name, dn_loc, homepage, '', '', ''])
+                except Exception as e:
+                    logger.warning("Failed to parse oss information:" + str(e))
 
         return sheet_list
+
+
+def get_checkout_dirname():
+    checkout_dir_list = []
+    if os.path.isdir(checkout_dir):
+        for item in os.listdir(checkout_dir):
+            if os.path.isdir(os.path.join(checkout_dir, item)):
+                checkout_dir_list.append(item)
+
+    return checkout_dir_list