Merge pull request #154 from SeongjunJo/exclude_path

Supports for excluding paths

Author: soimkim

requirements.txt

@@ -2,7 +2,7 @@ pyparsing
 scancode-toolkit==32.0.2
 scanoss
 XlsxWriter
-fosslight_util>=1.4.40
+fosslight_util>=1.4.43
 PyYAML
 wheel>=0.38.1
 intbitset

src/fosslight_source/_help.py

@@ -18,6 +18,7 @@
             -h\t\t\t   Print help message
             -v\t\t\t   Print FOSSLight Source Scanner version
             -m\t\t\t   Print additional information for scan result on separate sheets
+            -e <path>\t\t   Path to exclude from analysis (file and directory)
             -o <output_path>\t   Output path (Path or file name)
             -f <format>\t\t   Output file format (excel, csv, opossum, yaml)
         Options only for FOSSLight Source Scanner

src/fosslight_source/_parsing_scanoss_file.py

@@ -3,6 +3,7 @@
 # Copyright (c) 2020 LG Electronics Inc.
 # SPDX-License-Identifier: Apache-2.0
 
+import os
 import logging
 import fosslight_util.constant as constant
 from ._scan_item import ScanItem
@@ -34,10 +35,14 @@ def parsing_extraInfo(scanned_result):
     return scanoss_extra_info
 
 
-def parsing_scanResult(scanoss_report):
+def parsing_scanResult(scanoss_report, path_to_scan="", path_to_exclude=[]):
     scanoss_file_item = []
+    abs_path_to_exclude = [os.path.abspath(os.path.join(path_to_scan, path)) for path in path_to_exclude]
 
     for file_path, findings in scanoss_report.items():
+        abs_file_path = os.path.abspath(os.path.join(path_to_scan, file_path))
+        if any(os.path.commonpath([abs_file_path, exclude_path]) == exclude_path for exclude_path in abs_path_to_exclude):
+            continue
         result_item = ScanItem(file_path)
         if 'id' in findings[0]:
             if "none" == findings[0]['id']:

src/fosslight_source/cli.py

@@ -44,6 +44,7 @@ def main():
     _result_log = {}
 
     path_to_scan = os.getcwd()
+    path_to_exclude = []
     write_json_file = False
     output_file_name = ""
     print_matched_text = False
@@ -65,6 +66,7 @@ def main():
     parser.add_argument('-s', '--scanner', nargs=1, type=str, required=False, default='all')
     parser.add_argument('-t', '--timeout', type=int, required=False, default=120)
     parser.add_argument('-c', '--cores', type=int, required=False, default=-1)
+    parser.add_argument('-e', '--exclude', nargs='*', required=False, default=[])
     parser.add_argument('--no_correction', action='store_true', required=False)
     parser.add_argument('--correct_fpath', nargs=1, type=str, required=False)
 
@@ -78,6 +80,8 @@ def main():
         path_to_scan = os.getcwd()
     else:
         path_to_scan = ''.join(args.path)
+    if args.exclude:
+        path_to_exclude = args.exclude
     if args.json:
         write_json_file = True
     output_file_name = ''.join(args.output)
@@ -103,7 +107,8 @@ def main():
     if os.path.isdir(path_to_scan):
         result = []
         result = run_scanners(path_to_scan, output_file_name, write_json_file, core, True,
-                              print_matched_text, format, time_out, correct_mode, correct_filepath, selected_scanner)
+                              print_matched_text, format, time_out, correct_mode, correct_filepath,
+                              selected_scanner, path_to_exclude)
         _result_log["Scan Result"] = result[1]
 
         try:
@@ -115,9 +120,26 @@ def main():
         sys.exit(1)
 
 
+def count_files(path_to_scan, path_to_exclude):
+    total_files = 0
+    excluded_files = 0
+    abs_path_to_exclude = [os.path.abspath(os.path.join(path_to_scan, path)) for path in path_to_exclude]
+
+    for root, _, files in os.walk(path_to_scan):
+        for file in files:
+            file_path = os.path.join(root, file)
+            abs_file_path = os.path.abspath(file_path)
+            if any(os.path.commonpath([abs_file_path, exclude_path]) == exclude_path
+                   for exclude_path in abs_path_to_exclude):
+                excluded_files += 1
+            total_files += 1
+
+    return total_files, excluded_files
+
+
 def create_report_file(_start_time, merged_result, license_list, scanoss_result, selected_scanner, need_license=False,
                        output_path="", output_file="", output_extension="", correct_mode=True, correct_filepath="",
-                       path_to_scan=""):
+                       path_to_scan="", path_to_exclude=[]):
     """
     Create report files for given scanned result.
 
@@ -146,9 +168,10 @@ def create_report_file(_start_time, merged_result, license_list, scanoss_result,
 
     cover = CoverItem(tool_name=_PKG_NAME,
                       start_time=_start_time,
-                      input_path=path_to_scan)
-    files_count = sum([len(files) for r, d, files in os.walk(path_to_scan)])
-    cover.comment = f"Total number of files: {files_count} "
+                      input_path=path_to_scan,
+                      exclude_path=path_to_exclude)
+    files_count, removed_files_count = count_files(path_to_scan, path_to_exclude)
+    cover.comment = f"Total number of files / removed files: {files_count} / {removed_files_count}"
     if len(merged_result) == 0:
         if files_count < 1:
             cover.comment += "(No file detected.)"
@@ -224,7 +247,7 @@ def merge_results(scancode_result=[], scanoss_result=[], spdx_downloads={}):
 
 def run_scanners(path_to_scan, output_file_name="", write_json_file=False, num_cores=-1, called_by_cli=True,
                  print_matched_text=False, format="", time_out=120, correct_mode=True, correct_filepath="",
-                 selected_scanner='all'):
+                 selected_scanner='all', path_to_exclude=[]):
     """
     Run Scancode and scanoss.py for the given path.
 
@@ -252,7 +275,7 @@ def run_scanners(path_to_scan, output_file_name="", write_json_file=False, num_c
 
     success, msg, output_path, output_file, output_extension = check_output_format(output_file_name, format)
     logger, result_log = init_log(os.path.join(output_path, f"fosslight_log_src_{start_time}.txt"),
-                                  True, logging.INFO, logging.DEBUG, _PKG_NAME, path_to_scan)
+                                  True, logging.INFO, logging.DEBUG, _PKG_NAME, path_to_scan, path_to_exclude)
     if output_extension != '.xlsx' and output_extension and print_matched_text:
         logger.warning("-m option is only available for excel.")
         print_matched_text = False
@@ -261,14 +284,17 @@ def run_scanners(path_to_scan, output_file_name="", write_json_file=False, num_c
             success, result_log[RESULT_KEY], scancode_result, license_list = run_scan(path_to_scan, output_file_name,
                                                                                       write_json_file, num_cores, True,
                                                                                       print_matched_text, format, called_by_cli,
-                                                                                      time_out, correct_mode, correct_filepath)
+                                                                                      time_out, correct_mode, correct_filepath,
+                                                                                      path_to_exclude)
         if selected_scanner == 'scanoss' or selected_scanner == 'all' or selected_scanner == '':
-            scanoss_result = run_scanoss_py(path_to_scan, output_file_name, format, True, write_json_file, num_cores)
+            scanoss_result = run_scanoss_py(path_to_scan, output_file_name, format, True,
+                                            write_json_file, num_cores, path_to_exclude)
         if selected_scanner in SCANNER_TYPE:
-            spdx_downloads = get_spdx_downloads(path_to_scan)
+            spdx_downloads = get_spdx_downloads(path_to_scan, path_to_exclude)
             merged_result = merge_results(scancode_result, scanoss_result, spdx_downloads)
-            create_report_file(start_time, merged_result, license_list, scanoss_result, selected_scanner, print_matched_text,
-                               output_path, output_file, output_extension, correct_mode, correct_filepath, path_to_scan)
+            create_report_file(start_time, merged_result, license_list, scanoss_result, selected_scanner,
+                               print_matched_text, output_path, output_file, output_extension, correct_mode,
+                               correct_filepath, path_to_scan, path_to_exclude)
         else:
             print_help_msg_source_scanner()
             result_log[RESULT_KEY] = "Unsupported scanner"

src/fosslight_source/run_scancode.py

@@ -25,7 +25,7 @@
 
 def run_scan(path_to_scan, output_file_name="",
              _write_json_file=False, num_cores=-1, return_results=False, need_license=False, format="",
-             called_by_cli=False, time_out=120, correct_mode=True, correct_filepath=""):
+             called_by_cli=False, time_out=120, correct_mode=True, correct_filepath="", path_to_exclude=[]):
     if not called_by_cli:
         global logger
 
@@ -62,20 +62,35 @@ def run_scan(path_to_scan, output_file_name="",
 
         if not called_by_cli:
             logger, _result_log = init_log(os.path.join(output_path, f"fosslight_log_src_{_start_time}.txt"),
-                                           True, logging.INFO, logging.DEBUG, _PKG_NAME, path_to_scan)
+                                           True, logging.INFO, logging.DEBUG, _PKG_NAME, path_to_scan, path_to_exclude)
 
         num_cores = multiprocessing.cpu_count() - 1 if num_cores < 0 else num_cores
 
         if os.path.isdir(path_to_scan):
             try:
                 time_out = float(time_out)
+                pretty_params = {}
+                pretty_params["path_to_scan"] = path_to_scan
+                pretty_params["path_to_exclude"] = path_to_exclude
+                pretty_params["output_file"] = output_file_name
+                total_files_to_excluded = []
+                if path_to_exclude:
+                    for path in path_to_exclude:
+                        path = os.path.join(path_to_scan, path)
+                        if os.path.isdir(path):
+                            for root, _, files in os.walk(path):
+                                total_files_to_excluded.extend([os.path.normpath(os.path.join(root, file)).replace("\\", "/")
+                                                                for file in files])
+                        elif os.path.isfile(path):
+                            total_files_to_excluded.append(os.path.normpath(path).replace("\\", "/"))
+
                 rc, results = cli.run_scan(path_to_scan, max_depth=100,
                                            strip_root=True, license=True,
                                            copyright=True, return_results=True,
-                                           processes=num_cores,
-                                           output_json_pp=output_json_file,
-                                           only_findings=True, license_text=True,
-                                           url=True, timeout=time_out)
+                                           processes=num_cores, pretty_params=pretty_params,
+                                           output_json_pp=output_json_file, only_findings=True,
+                                           license_text=True, url=True, timeout=time_out,
+                                           include=(), ignore=tuple(total_files_to_excluded))
 
                 if not rc:
                     msg = "Source code analysis failed."

src/fosslight_source/run_scanoss.py

@@ -29,7 +29,8 @@ def get_scanoss_extra_info(scanned_result):
     return parsing_extraInfo(scanned_result)
 
 
-def run_scanoss_py(path_to_scan, output_file_name="", format="", called_by_cli=False, write_json_file=False, num_threads=-1):
+def run_scanoss_py(path_to_scan, output_file_name="", format="", called_by_cli=False,
+                   write_json_file=False, num_threads=-1, path_to_exclude=[]):
     """
     Run scanoss.py for the given path.
 
@@ -46,7 +47,7 @@ def run_scanoss_py(path_to_scan, output_file_name="", format="", called_by_cli=F
         global logger
         _start_time = datetime.now().strftime('%y%m%d_%H%M')
         logger, _result_log = init_log(os.path.join(output_path, f"fosslight_log_src_{_start_time}.txt"),
-                                       True, logging.INFO, logging.DEBUG, _PKG_NAME, path_to_scan)
+                                       True, logging.INFO, logging.DEBUG, _PKG_NAME, path_to_scan, path_to_exclude)
 
     scanoss_file_list = []
     try:
@@ -73,9 +74,30 @@ def run_scanoss_py(path_to_scan, output_file_name="", format="", called_by_cli=F
     try:
         os.system(scan_command)
         if os.path.isfile(output_json_file):
+            total_files_to_excluded = []
+            if path_to_exclude:
+                for path in path_to_exclude:
+                    path = os.path.join(path_to_scan, path)
+                    if os.path.isdir(path):
+                        for root, _, files in os.walk(path):
+                            root = root[len(path_to_scan) + 1:]
+                            total_files_to_excluded.extend([os.path.normpath(os.path.join(root, file)).replace('\\', '/')
+                                                            for file in files])
+                    elif os.path.isfile(path):
+                        path = path[len(path_to_scan) + 1:]
+                        total_files_to_excluded.append(os.path.normpath(path).replace('\\', '/'))
+
+            with open(output_json_file, "r") as st_json:
+                st_python = json.load(st_json)
+                for key_to_exclude in total_files_to_excluded:
+                    if key_to_exclude in st_python:
+                        del st_python[key_to_exclude]
+            with open(output_json_file, 'w') as st_json:
+                json.dump(st_python, st_json, indent=4)
             with open(output_json_file, "r") as st_json:
                 st_python = json.load(st_json)
-                scanoss_file_list = parsing_scanResult(st_python)
+                scanoss_file_list = parsing_scanResult(st_python, path_to_scan, path_to_exclude)
+
     except Exception as error:
         logger.debug(f"SCANOSS Parsing {path_to_scan}: {error}")
 

src/fosslight_source/run_spdx_extractor.py

@@ -12,19 +12,25 @@
 logger = logging.getLogger(constant.LOGGER_NAME)
 
 
-def get_file_list(path_to_scan):
+def get_file_list(path_to_scan, path_to_exclude=[]):
     file_list = []
+    abs_path_to_exclude = [os.path.abspath(os.path.join(path_to_scan, path)) for path in path_to_exclude]
     for root, dirs, files in os.walk(path_to_scan):
         for file in files:
-            file_list.append(os.path.join(root, file))
+            file_path = os.path.join(root, file)
+            abs_file_path = os.path.abspath(file_path)
+            if any(os.path.commonpath([abs_file_path, exclude_path]) == exclude_path
+                    for exclude_path in abs_path_to_exclude):
+                continue
+            file_list.append(file_path)
     return file_list
 
 
-def get_spdx_downloads(path_to_scan):
+def get_spdx_downloads(path_to_scan, path_to_exclude=[]):
     download_dict = {}
     find_word = re.compile(rb"SPDX-PackageDownloadLocation\s*:\s*(\S+)", re.IGNORECASE)
 
-    file_list = get_file_list(path_to_scan)
+    file_list = get_file_list(path_to_scan, path_to_exclude)
 
     for file in file_list:
         try:

tox.ini

@@ -27,6 +27,7 @@ deps =
 commands =
   rm -rf test_scan
   fosslight_source -p tests/test_files -j -m -o test_scan
+  fosslight_source -p tests -e test_files/test cli_test.py -j -m -o test_scan2
 
 [testenv:release]
 deps =
@@ -38,8 +39,11 @@ commands =
   fosslight_source -p tests/test_files -o test_scan/scan_result.csv
   cat test_scan/scan_result.csv
 
-  fosslight_source -p tests/test_files -m -j -o test_scan2/
-  ls test_scan2/
+  fosslight_source -p tests -e test_files/test cli_test.py -j -m -o test_scan2/scan_exclude_result.csv
+  cat test_scan2/scan_exclude_result.csv
+
+  fosslight_source -p tests/test_files -m -j -o test_scan3/
+  ls test_scan3/
 
   python tests/cli_test.py
   pytest -v --flake8