Add dependencies of cocoapods, go, nuget package in comment (#130)

* Add dependencies of cocoapods, go, nuget package in comment
* Fix the android gradle allDeps task exception bug

---------

Signed-off-by: Jiyeong Seok <[email protected]>

Author: dd-jy

.github/workflows/publish-release.yml

@@ -85,7 +85,7 @@ jobs:
         asset_content_type: application/octet-stream
 
   deploy:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     needs: build
     steps:
     - uses: actions/checkout@v3

src/fosslight_dependency/_package_manager.py

@@ -91,19 +91,24 @@ def run_gradle_task(self):
             shutil.copy(const.SUPPORT_PACKAE.get(self.package_manager_name), gradle_backup)
             ret = self.add_allDeps_in_gradle()
             if ret:
-                if os.path.isfile('gradlew') or os.path.isfile('gradlew.bat'):
-                    if self.platform == const.WINDOWS:
-                        cmd_gradle = "gradlew.bat"
-                    else:
-                        cmd_gradle = "./gradlew"
-
-                    cmd = f"{cmd_gradle} allDeps"
-                    ret = subprocess.check_output(cmd, shell=True, encoding='utf-8')
-                    if ret != 0:
-                        self.parse_dependency_tree(ret)
-                    else:
-                        self.set_direct_dependencies(False)
-                        logger.warning("Failed to run allDeps task.")
+                try:
+                    if os.path.isfile('gradlew') or os.path.isfile('gradlew.bat'):
+                        if self.platform == const.WINDOWS:
+                            cmd_gradle = "gradlew.bat"
+                        else:
+                            cmd_gradle = "./gradlew"
+
+                        cmd = f"{cmd_gradle} allDeps"
+                        ret = subprocess.check_output(cmd, shell=True, encoding='utf-8')
+                        if ret != 0:
+                            self.parse_dependency_tree(ret)
+                        else:
+                            self.set_direct_dependencies(False)
+                            logger.warning("Failed to run allDeps task.")
+                except Exception as e:
+                    self.set_direct_dependencies(False)
+                    logger.error(f'Fail to run allDeps: {e}')
+                    logger.warning('It cannot print the direct/transitive dependencies relationship.')
 
             if os.path.isfile(gradle_backup):
                 os.remove(const.SUPPORT_PACKAE.get(self.package_manager_name))

src/fosslight_dependency/package_manager/Cocoapods.py

@@ -34,8 +34,6 @@ def parse_oss_information(self, f_name):
         with open(f_name, 'r', encoding='utf8') as input_fp:
             podfile_yaml = yaml.load(input_fp, Loader=yaml.FullLoader)
 
-        pod_in_sepc_list = []
-        pod_not_in_spec_list = []
         spec_repo_list = []
         external_source_list = []
         comment = ''
@@ -49,47 +47,64 @@ def parse_oss_information(self, f_name):
                 external_source_list.append(external_sources_key)
                 spec_repo_list.append(external_sources_key)
         if len(spec_repo_list) == 0:
-            logger.error("Cannot fint SPEC REPOS or EXTERNAL SOURCES in Podfile.lock.")
+            logger.error("Cannot find SPEC REPOS or EXTERNAL SOURCES in Podfile.lock.")
             return ''
 
         for dep_key in podfile_yaml[_dependencies]:
             dep_key_re = re.findall(r'(^\S*)', dep_key)
-            dep_name = dep_key_re[0]
-            if '/' in dep_name:
-                dep_name = dep_name.split('/')[0]
-            self.direct_dep_list.append(dep_name)
-
-        for pods_list in podfile_yaml['PODS']:
-            if not isinstance(pods_list, str):
-                for pods_list_key, pods_list_item in pods_list.items():
-                    pod_in_sepc_list, spec_repo_list, pod_not_in_spec_list = \
-                        compile_pods_item(pods_list_key, spec_repo_list, pod_in_sepc_list, pod_not_in_spec_list)
+            self.direct_dep_list.append(dep_key_re[0])
+
+        pod_item_list = {}
+        for pods_i in podfile_yaml['PODS']:
+            if not isinstance(pods_i, str):
+                for key, items in pods_i.items():
+                    k_name, k_ver = get_pods_info(key)
+                    pod_item_list[k_name] = k_ver
+                    self.relation_tree[f'{k_name}({k_ver})'] = []
+                    for item in items:
+                        i_name, _ = get_pods_info(item)
+                        self.relation_tree[f'{k_name}({k_ver})'].append(i_name)
             else:
-                pod_in_sepc_list, spec_repo_list, pod_not_in_spec_list = \
-                    compile_pods_item(pods_list, spec_repo_list, pod_in_sepc_list, pod_not_in_spec_list)
+                oss_name, oss_version = get_pods_info(pods_i)
+                pod_item_list[oss_name] = oss_version
 
-        if len(spec_repo_list) != 0:
-            for spec_in_item in spec_repo_list:
-                spec_oss_name_adding_core = spec_in_item + "/Core"
-                for pod_not_item in pod_not_in_spec_list:
-                    if spec_oss_name_adding_core == pod_not_item[0]:
-                        pod_in_sepc_list.append([spec_in_item, pod_not_item[1]])
+        for rel_key in self.relation_tree:
+            try:
+                tmp_item_list = []
+                for ri in self.relation_tree[rel_key]:
+                    ri_version = pod_item_list[ri]
+                    tmp_item_list.append(f'{ri}({ri_version})')
+                self.relation_tree[rel_key] = []
+                self.relation_tree[rel_key].extend(tmp_item_list)
+            except Exception as e:
+                logger.warning(f'Fail to check packages of {rel_key}: {e}')
+                if rel_key in self.relation_tree:
+                    self.relation_tree[rel_key] = []
 
         sheet_list = []
-
-        for pod_oss in pod_in_sepc_list:
+        for pod_oss_name_origin, pod_oss_version in pod_item_list.items():
             try:
+                comment_list = []
                 if self.direct_dep:
-                    if pod_oss[0] in self.direct_dep_list:
-                        comment = 'direct'
+                    if pod_oss_name_origin in self.direct_dep_list:
+                        comment_list.append('direct')
                     else:
-                        comment = 'transitive'
-                if pod_oss[0] in external_source_list:
-                    podspec_filename = pod_oss[0] + '.podspec.json'
+                        comment_list.append('transitive')
+                    if f'{pod_oss_name_origin}({oss_version})' in self.relation_tree:
+                        rel_items = [f'{self.package_manager_name}:{ri}'
+                                     for ri in self.relation_tree[f'{pod_oss_name_origin}({oss_version})']]
+                        comment_list.extend(rel_items)
+                comment = ', '.join(comment_list)
+
+                pod_oss_name = pod_oss_name_origin
+                if '/' in pod_oss_name_origin:
+                    pod_oss_name = pod_oss_name_origin.split('/')[0]
+                if pod_oss_name in external_source_list:
+                    podspec_filename = pod_oss_name + '.podspec.json'
                     spec_file_path = os.path.join("Pods", "Local Podspecs", podspec_filename)
                 else:
                     search_oss_name = ""
-                    for alphabet_oss in pod_oss[0]:
+                    for alphabet_oss in pod_oss_name:
                         if not alphabet_oss.isalnum():
                             search_oss_name += f"\\\\{alphabet_oss}"
                         else:
@@ -106,16 +121,19 @@ def parse_oss_information(self, f_name):
                         file_path_without_version = os.path.join(os.sep, *file_path[:-2])
                     else:
                         file_path_without_version = os.path.join(*file_path[:-2])
-                    spec_file_path = os.path.join(file_path_without_version, pod_oss[1], file_path[-1])
+                    spec_file_path = os.path.join(file_path_without_version, pod_oss_version, file_path[-1])
 
                 oss_name, oss_version, license_name, dn_loc, homepage = self.get_oss_in_podspec(spec_file_path)
                 if oss_name == '':
                     continue
-
+                if pod_oss_version != oss_version:
+                    logger.warning(f'{pod_oss_name_origin} has different version({pod_oss_version})\
+                                   with spec version({oss_version})')
                 sheet_list.append([const.SUPPORT_PACKAE.get(self.package_manager_name),
-                                  oss_name, oss_version, license_name, dn_loc, homepage, '', '', comment])
+                                  f'{self.package_manager_name}:{pod_oss_name_origin}',
+                                   pod_oss_version, license_name, dn_loc, homepage, '', '', comment])
             except Exception as e:
-                logger.warning(f"Fail to get {pod_oss[0]}:{e}")
+                logger.warning(f"Fail to get {pod_oss_name_origin}:{e}")
 
         return sheet_list
 
@@ -159,20 +177,10 @@ def parse_direct_dependencies(self):
         self.direct_dep = True
 
 
-def compile_pods_item(pods_item, spec_repo_list, pod_in_sepc_list, pod_not_in_spec_list):
-    pods_item_re = re.findall(r'(\S*)\s{1}\((.*)\)', pods_item)
+def get_pods_info(pods_item):
+    pods_item_re = re.findall(r'(\S*)(?:\s{1}\((.*)\))?', pods_item)
 
     oss_name = pods_item_re[0][0]
     oss_version = pods_item_re[0][1]
 
-    oss_info = []
-    oss_info.append(oss_name)
-    oss_info.append(oss_version)
-
-    if oss_name in spec_repo_list:
-        pod_in_sepc_list.append(oss_info)
-        spec_repo_list.remove(oss_name)
-    else:
-        pod_not_in_spec_list.append(oss_info)
-
-    return pod_in_sepc_list, spec_repo_list, pod_not_in_spec_list
+    return oss_name, oss_version

src/fosslight_dependency/package_manager/Go.py

@@ -9,6 +9,7 @@
 import json
 from bs4 import BeautifulSoup
 import urllib.request
+import re
 import fosslight_util.constant as constant
 import fosslight_dependency.constant as const
 from fosslight_dependency._package_manager import PackageManager
@@ -33,6 +34,18 @@ def __del__(self):
         if os.path.isfile(self.tmp_file_name):
             os.remove(self.tmp_file_name)
 
+    def parse_dependency_tree(self, go_deptree_txt):
+        for line in go_deptree_txt.split('\n'):
+            re_result = re.findall(r'(\S+)@v(\S+)\s(\S+)@v(\S+)', line)
+            if len(re_result) > 0 and len(re_result[0]) >= 4:
+                oss_name = re_result[0][0]
+                oss_ver = re_result[0][1]
+                pkg_name = re_result[0][2]
+                pkg_ver = re_result[0][3]
+                if f'{oss_name}({oss_ver})' not in self.relation_tree:
+                    self.relation_tree[f'{oss_name}({oss_ver})'] = []
+                self.relation_tree[f'{oss_name}({oss_ver})'].append(f'{pkg_name}({pkg_ver})')
+
     def run_plugin(self):
         ret = True
 
@@ -46,6 +59,11 @@ def run_plugin(self):
 
         self.append_input_package_list_file(self.tmp_file_name)
 
+        cmd_tree = "go mod graph"
+        ret_cmd_tree = subprocess.check_output(cmd_tree, shell=True, text=True, encoding='utf-8')
+        if ret_cmd_tree != 0:
+            self.parse_dependency_tree(ret_cmd_tree)
+
         return ret
 
     def parse_oss_information(self, f_name):
@@ -67,40 +85,44 @@ def parse_oss_information(self, f_name):
                         continue
                 package_path = dep_item['Path']
                 oss_name = f"{self.package_manager_name}:{package_path}"
-                oss_version = dep_item['Version']
+                oss_origin_version = dep_item['Version']
+                if oss_origin_version.startswith('v'):
+                    oss_version = oss_origin_version[1:]
 
-                comment = []
+                comment_list = []
                 if self.direct_dep:
                     if indirect in dep_item:
                         if dep_item[indirect]:
-                            comment.append('transitive')
+                            comment_list.append('transitive')
                         else:
-                            comment.append('direct')
+                            comment_list.append('direct')
                     else:
-                        comment.append('direct')
+                        comment_list.append('direct')
+
+                if f'{package_path}({oss_version})' in self.relation_tree:
+                    rel_items = [f'{self.package_manager_name}:{ri}'
+                                 for ri in self.relation_tree[f'{package_path}({oss_version})']]
+                    comment_list.extend(rel_items)
 
                 homepage_set = []
                 homepage = self.dn_url + package_path
 
-                if oss_version:
-                    tmp_homepage = f"{homepage}@{oss_version}"
+                if oss_origin_version:
+                    tmp_homepage = f"{homepage}@{oss_origin_version}"
                     homepage_set.append(tmp_homepage)
                 homepage_set.append(homepage)
 
                 license_name = ''
                 dn_loc = ''
 
-                if oss_version.startswith('v'):
-                    oss_version = oss_version[1:]
-
                 for homepage_i in homepage_set:
                     try:
                         res = urllib.request.urlopen(homepage_i)
                         if res.getcode() == 200:
                             urlopen_success = True
                             if homepage_i == homepage:
                                 if oss_version:
-                                    comment.append(f'Cannot connect {tmp_homepage}, get info from the latest version.')
+                                    comment_list.append(f'Cannot connect {tmp_homepage}, get info from the latest version.')
                             break
                     except Exception:
                         continue
@@ -123,7 +145,7 @@ def parse_oss_information(self, f_name):
                 logging.warning(f"Fail to parse {package_path} in go mod : {e}")
                 continue
 
-            comment = ', '.join(comment)
+            comment = ', '.join(comment_list)
             sheet_list.append([const.SUPPORT_PACKAE.get(self.package_manager_name),
                               oss_name, oss_version, license_name, dn_loc, homepage, '', '', comment])
 

src/fosslight_dependency/package_manager/Nuget.py

@@ -24,6 +24,7 @@ class Nuget(PackageManager):
     dn_url = "https://nuget.org/packages/"
     packageReference = False
     nuget_api_url = 'https://api.nuget.org/v3-flatcontainer/'
+    dotnet_ver = []
 
     def __init__(self, input_dir, output_dir):
         super().__init__(self.package_manager_name, self.dn_url, input_dir, output_dir)
@@ -40,14 +41,13 @@ def parse_oss_information(self, f_name):
             sheet_list = []
             package_list = []
             if self.packageReference:
-                package_list = self.get_package_list_in_packages_assets(input_fp)
-                self.get_direct_package_in_packagereference()
+                package_list = self.get_package_info_in_packagereference(input_fp)
             else:
                 package_list = self.get_package_list_in_packages_config(input_fp)
 
         for oss_origin_name, oss_version in package_list:
             try:
-                oss_name = self.package_manager_name + ":" + oss_origin_name
+                oss_name = f'{self.package_manager_name}:{oss_origin_name}'
 
                 comment = []
                 dn_loc = ''
@@ -105,8 +105,13 @@ def parse_oss_information(self, f_name):
                     else:
                         comment.append('transitive')
 
+                    if f'{oss_origin_name}({oss_version})' in self.relation_tree:
+                        rel_items = [f'{self.package_manager_name}:{ri}'
+                                     for ri in self.relation_tree[f'{oss_origin_name}({oss_version})']]
+                        comment.extend(rel_items)
+
                 sheet_list.append([','.join(self.input_package_list_file),
-                                  oss_name, oss_version, license_name, dn_loc, homepage, '', '', ','.join(comment)])
+                                  oss_name, oss_version, license_name, dn_loc, homepage, '', '', ', '.join(comment)])
 
             except Exception as e:
                 logger.warning(f"Failed to parse oss information: {e}")
@@ -123,15 +128,50 @@ def get_package_list_in_packages_config(self, input_fp):
             package_list.append([p.get("id"), p.get("version")])
         return package_list
 
-    def get_package_list_in_packages_assets(self, input_fp):
-        package_list = []
+    def get_package_info_in_packagereference(self, input_fp):
         json_f = json.load(input_fp)
+
+        self.get_dotnet_ver_list(json_f)
+        package_list = self.get_package_list_in_packages_assets(json_f)
+        self.get_dependency_tree(json_f)
+        self.get_direct_package_in_packagereference()
+
+        return package_list
+
+    def get_package_list_in_packages_assets(self, json_f):
+        package_list = []
         for item in json_f['libraries']:
             if json_f['libraries'][item]['type'] == 'package':
                 oss_info = item.split('/')
                 package_list.append([oss_info[0], oss_info[1]])
         return package_list
 
+    def get_dotnet_ver_list(self, json_f):
+        json_project_group = json_f['projectFileDependencyGroups']
+        for dotnet_ver in json_project_group:
+            self.dotnet_ver.append(dotnet_ver)
+
+    def get_dependency_tree(self, json_f):
+        json_target = json_f['targets']
+        for item in json_target:
+            if item not in self.dotnet_ver:
+                continue
+            json_item = json_target[item]
+            for pkg in json_item:
+                json_pkg = json_item[pkg]
+                if 'type' not in json_pkg:
+                    continue
+                if 'dependencies' not in json_pkg:
+                    continue
+                if json_pkg['type'] != 'package':
+                    continue
+                oss_info = pkg.split('/')
+                self.relation_tree[f'{oss_info[0]}({oss_info[1]})'] = []
+                for dep in json_pkg['dependencies']:
+                    oss_name = dep
+                    oss_ver = json_pkg['dependencies'][dep]
+                    self.relation_tree[f'{oss_info[0]}({oss_info[1]})'].append(f'{oss_name}({oss_ver})')
+
     def get_direct_package_in_packagereference(self):
         for f in os.listdir(self.input_dir):
             if os.path.isfile(f) and ((f.split('.')[-1] == 'csproj') or (f.split('.')[-1] == 'xproj')):