Merge pull request #83 from fosslight/develop

Support to comment direct/transitive type

Author: dd-jy

.reuse/dep5

@@ -92,4 +92,8 @@ License: Apache-2.0
 
 Files: tests/test_swift2/*
 Copyright: 2022 LG Electronics
-License: Apache-2.0
+License: Apache-2.0
+
+Files: tests/test_gradle2/*
+Copyright: 2014 Netflix
+License: Apache-2.0

requirements.txt

@@ -6,3 +6,4 @@ pyyaml
 lastversion
 fosslight_util>=1.3.12
 PyGithub
+requirements-parser

src/fosslight_dependency/_analyze_dependency.py

@@ -22,7 +22,8 @@
 
 
 def analyze_dependency(package_manager_name, input_dir, output_dir, pip_activate_cmd='', pip_deactivate_cmd='',
-                       output_custom_dir='', app_name=const.default_app_name, github_token='', manifest_file_name=[]):
+                       output_custom_dir='', app_name=const.default_app_name, github_token='', manifest_file_name=[],
+                       direct=True):
     ret = True
     package_sheet_list = []
 
@@ -53,8 +54,14 @@ def analyze_dependency(package_manager_name, input_dir, output_dir, pip_activate
 
     if manifest_file_name:
         package_manager.set_manifest_file(manifest_file_name)
+
+    if direct:
+        package_manager.set_direct_dependencies(direct)
     ret = package_manager.run_plugin()
     if ret:
+        if direct:
+            package_manager.parse_direct_dependencies()
+
         for f_name in package_manager.input_package_list_file:
             logger.info(f"Parse oss information with file: {f_name}")
 
@@ -69,4 +76,6 @@ def analyze_dependency(package_manager_name, input_dir, output_dir, pip_activate
     else:
         logger.error(f"### Fail to analyze: {package_manager_name}")
 
+    del package_manager
+
     return ret, package_sheet_list

src/fosslight_dependency/_package_manager.py

@@ -9,6 +9,9 @@
 import platform
 import re
 import base64
+import subprocess
+import shutil
+import copy
 import fosslight_util.constant as constant
 import fosslight_dependency.constant as const
 
@@ -27,9 +30,15 @@
 
 class PackageManager:
     input_package_list_file = []
+    direct_dep = False
+    total_dep_list = []
+    direct_dep_list = []
 
     def __init__(self, package_manager_name, dn_url, input_dir, output_dir):
         self.input_package_list_file = []
+        self.direct_dep = False
+        self.total_dep_list = []
+        self.direct_dep_list = []
         self.package_manager_name = package_manager_name
         self.input_dir = input_dir
         self.output_dir = output_dir
@@ -39,8 +48,22 @@ def __init__(self, package_manager_name, dn_url, input_dir, output_dir):
         self.platform = platform.system()
         self.license_scanner_bin = check_license_scanner(self.platform)
 
+    def __del__(self):
+        self.input_package_list_file = []
+        self.direct_dep = False
+        self.total_dep_list = []
+        self.direct_dep_list = []
+        self.package_manager_name = ''
+        self.input_dir = ''
+        self.output_dir = ''
+        self.dn_url = ''
+        self.manifest_file_name = []
+
     def run_plugin(self):
-        logger.info(f"This package manager({self.package_manager_name}) skips the step to run plugin.")
+        if self.package_manager_name == const.GRADLE or self.package_manager_name == const.ANDROID:
+            self.run_gradle_task()
+        else:
+            logger.info(f"This package manager({self.package_manager_name}) skips the step to run plugin.")
         return True
 
     def append_input_package_list_file(self, input_package_file):
@@ -49,6 +72,86 @@ def append_input_package_list_file(self, input_package_file):
     def set_manifest_file(self, manifest_file_name):
         self.manifest_file_name = manifest_file_name
 
+    def set_direct_dependencies(self, direct):
+        self.direct_dep = direct
+
+    def parse_direct_dependencies(self):
+        pass
+
+    def run_gradle_task(self):
+        dependency_tree_fname = 'tmp_dependency_tree.txt'
+        if os.path.isfile(const.SUPPORT_PACKAE.get(self.package_manager_name)):
+            gradle_backup = f'{const.SUPPORT_PACKAE.get(self.package_manager_name)}_bk'
+
+            shutil.copy(const.SUPPORT_PACKAE.get(self.package_manager_name), gradle_backup)
+            ret = self.add_allDeps_in_gradle()
+            if not ret:
+                return
+
+            ret = self.exeucte_gradle_task(dependency_tree_fname)
+            if ret != 0:
+                self.set_direct_dependencies(False)
+                logger.warning("Failed to run allDeps task.")
+            else:
+                self.parse_dependency_tree(dependency_tree_fname)
+
+            if os.path.isfile(dependency_tree_fname):
+                os.remove(dependency_tree_fname)
+
+            if os.path.isfile(gradle_backup):
+                os.remove(const.SUPPORT_PACKAE.get(self.package_manager_name))
+                shutil.move(gradle_backup, const.SUPPORT_PACKAE.get(self.package_manager_name))
+
+    def add_allDeps_in_gradle(self):
+        ret = False
+        configuration = 'project.configurations.runtimeClasspath, project.configurations.runtime'
+        if self.package_manager_name == 'android':
+            configuration = 'project.configurations.releaseRuntimeClasspath'
+
+        allDeps = f'''allprojects {{
+                   task allDeps(type: DependencyReportTask) {{
+                        doFirst{{
+                            try {{
+                                configurations = [{configuration}] as Set }}
+                            catch(UnknownConfigurationException) {{}}
+                        }}
+                    }}
+                    }}'''
+        try:
+            with open(const.SUPPORT_PACKAE.get(self.package_manager_name), 'a', encoding='utf8') as f:
+                f.write(allDeps)
+                ret = True
+        except Exception as e:
+            logging.warning(f"Cannot add the allDeps task in build.gradle: {e}")
+
+        return ret
+
+    def exeucte_gradle_task(self, dependency_tree_fname):
+        if os.path.isfile('gradlew') or os.path.isfile('gradlew.bat'):
+            if self.platform == const.WINDOWS:
+                cmd_gradle = "gradlew.bat"
+            else:
+                cmd_gradle = "./gradlew"
+        else:
+            return 1
+        cmd = f"{cmd_gradle} allDeps > {dependency_tree_fname}"
+
+        ret = subprocess.call(cmd, shell=True)
+        return ret
+
+    def parse_dependency_tree(self, f_name):
+        with open(f_name, 'r', encoding='utf8') as input_fp:
+            for i, line in enumerate(input_fp.readlines()):
+                try:
+                    line_bk = copy.deepcopy(line)
+                    re_result = re.findall(r'\-\-\-\s([^\:\s]+\:[^\:\s]+)\:([^\:\s]+)', line)
+                    if re_result:
+                        self.total_dep_list.append(re_result[0][0])
+                        if re.match(r'^[\+|\\]\-\-\-\s([^\:\s]+\:[^\:\s]+)\:([^\:\s]+)', line_bk):
+                            self.direct_dep_list.append(re_result[0][0])
+                except Exception as e:
+                    logger.error(f"Failed to parse dependency tree: {e}")
+
 
 def version_refine(oss_version):
     version_cmp = oss_version.upper()

src/fosslight_dependency/package_manager/Android.py

@@ -38,6 +38,7 @@ def parse_oss_information(self, f_name):
             sheet_list = []
 
             for i, line in enumerate(input_fp.readlines()):
+                comment = ''
                 split_str = line.strip().split("\t")
                 if i < 2:
                     continue
@@ -48,6 +49,17 @@ def parse_oss_information(self, f_name):
                     idx, manifest_file, oss_name, oss_version, license_name, dn_loc, homepage = split_str
                 else:
                     continue
-                sheet_list.append([manifest_file, oss_name, oss_version, license_name, dn_loc, homepage, '', '', ''])
+
+                if self.total_dep_list:
+                    if oss_name not in self.total_dep_list:
+                        continue
+
+                if self.direct_dep:
+                    if oss_name in self.direct_dep_list:
+                        comment = 'direct'
+                    else:
+                        comment = 'transitive'
+
+                sheet_list.append([manifest_file, oss_name, oss_version, license_name, dn_loc, homepage, '', '', comment])
 
         return sheet_list

src/fosslight_dependency/package_manager/Carthage.py

@@ -36,6 +36,7 @@ def __init__(self, input_dir, output_dir, github_token):
     def parse_oss_information(self, f_name):
         github = "github"
         checkout_dir_list = get_checkout_dirname()
+        comment = ''
         with open(f_name, 'r', encoding='utf8') as input_fp:
             sheet_list = []
             g = ''
@@ -89,14 +90,36 @@ def parse_oss_information(self, f_name):
                                 logger.warning(f"Failed to get license with github api: {e}")
                                 license_name == ''
 
+                    if self.direct_dep_list:
+                        if oss_origin_name in self.direct_dep_list:
+                            comment = 'direct'
+                        else:
+                            comment = 'transitive'
+
                     sheet_list.append([const.SUPPORT_PACKAE.get(self.package_manager_name),
-                                      oss_name, oss_version, license_name, dn_loc, homepage, '', '', ''])
+                                      oss_name, oss_version, license_name, dn_loc, homepage, '', '', comment])
 
                 except Exception as e:
                     logger.warning(f"Failed to parse oss information: {e}")
 
         return sheet_list
 
+    def parse_direct_dependencies(self):
+        self.direct_dep = True
+        cartfile = 'Cartfile'
+        if os.path.exists(cartfile):
+            with open(cartfile, 'r', encoding='utf8') as input_fp:
+                for i, line in enumerate(input_fp.readlines()):
+                    re_result = re.findall(r'(github|git)[\s]\"(\S*)\"[\s]\"(\S*)\"', line)
+                    try:
+                        oss_path = re_result[0][1]
+                        if oss_path.endswith('.git'):
+                            oss_path = oss_path[:-4]
+                        oss_origin_name = oss_path.split('/')[-1]
+                        self.direct_dep_list.append(oss_origin_name)
+                    except Exception as e:
+                        logger.warning(f"Failed to parse Cartfile: {e}")
+
 
 def get_checkout_dirname():
     checkout_dir_list = []

src/fosslight_dependency/package_manager/Cocoapods.py

@@ -17,6 +17,7 @@
 
 _spec_repos = 'SPEC REPOS'
 _external_sources = 'EXTERNAL SOURCES'
+_dependencies = 'DEPENDENCIES'
 _source_type = ['git', 'http', 'svn', 'hg']
 
 
@@ -38,6 +39,7 @@ def parse_oss_information(self, f_name):
         pod_not_in_spec_list = []
         spec_repo_list = []
         external_source_list = []
+        comment = ''
 
         if _spec_repos in podfile_yaml:
             for spec_item_key in podfile_yaml[_spec_repos]:
@@ -51,6 +53,13 @@ def parse_oss_information(self, f_name):
             logger.error("Cannot fint SPEC REPOS or EXTERNAL SOURCES in Podfile.lock.")
             return ''
 
+        for dep_key in podfile_yaml[_dependencies]:
+            dep_key_re = re.findall(r'(^\S*)', dep_key)
+            dep_name = dep_key_re[0]
+            if '/' in dep_name:
+                dep_name = dep_name.split('/')[0]
+            self.direct_dep_list.append(dep_name)
+
         for pods_list in podfile_yaml['PODS']:
             if not isinstance(pods_list, str):
                 for pods_list_key, pods_list_item in pods_list.items():
@@ -71,6 +80,11 @@ def parse_oss_information(self, f_name):
 
         for pod_oss in pod_in_sepc_list:
             try:
+                if self.direct_dep:
+                    if pod_oss[0] in self.direct_dep_list:
+                        comment = 'direct'
+                    else:
+                        comment = 'transitive'
                 if pod_oss[0] in external_source_list:
                     podspec_filename = pod_oss[0] + '.podspec.json'
                     spec_file_path = os.path.join("Pods", "Local Podspecs", podspec_filename)
@@ -98,7 +112,7 @@ def parse_oss_information(self, f_name):
                 oss_name, oss_version, license_name, dn_loc, homepage = self.get_oss_in_podspec(spec_file_path)
 
                 sheet_list.append([const.SUPPORT_PACKAE.get(self.package_manager_name),
-                                  oss_name, oss_version, license_name, dn_loc, homepage, '', '', ''])
+                                  oss_name, oss_version, license_name, dn_loc, homepage, '', '', comment])
             except Exception as e:
                 logger.warning(f"It failed to get {pod_oss[0]}:{e}")
                 logger.warning(traceback.format_exc())
@@ -131,6 +145,9 @@ def get_oss_in_podspec(self, spec_file_path):
 
         return oss_name, oss_version, license_name, dn_loc, homepage
 
+    def parse_direct_dependencies(self):
+        self.direct_dep = True
+
 
 def compile_pods_item(pods_item, spec_repo_list, pod_in_sepc_list, pod_not_in_spec_list):
     pods_item_re = re.findall(r'(\S*)\s{1}\((.*)\)', pods_item)