Merge pull request #10 from fosslight/develop

Add the -f option and change way to create output

Author: soimkim

README.md

@@ -55,19 +55,19 @@ FOSSLight Scanner is run with the **fosslight** command.
 ``` 
     -h                        Print help message
     -r                        Keep raw data 
-    -s <source_path>          Path to analyze source
+    -p <path>                 Path to analyze source
     -w <link>                 Link to be analyzaed can be downloaded by wget or git clone
     -o <output>               Output Directory or file
-    -d <dependency_path>      Path to analyze dependencies
-    -d <dependency_path> -a <additional_arg> (Using with -d option) Additional arguments for running dependency analysis  
+    -f <format>               Output file format (excel, csv, opossum)
+    -d <additional_arg>       Additional arguments for running dependency analysis 
 ```
 - Ref. Additional arguments for running dependency analysis. See the [FOSSLight Dependency Guide][fd_guide] for instructions.
 
 [fd_guide]: https://fosslight.org/fosslight-guide-en/scanner/2_dependency.html
 
 ### Ex 1. Local Source Analysis
 ```
-$ fosslight -d /home/source_path -a "-a 'source /test/Projects/venv/bin/activate' -d 'deactivate'" -s /home/source_path
+$ fosslight -p /home/source_path -a "-a 'source /test/Projects/venv/bin/activate' -d 'deactivate'"
 ```
 
 ### Ex 2. Download Link and analyze

requirements.txt

@@ -5,5 +5,5 @@ xlrd
 openpyxl
 progress
 pyyaml
-fosslight_util>=1.3.3
-fosslight_dependency>=3.7.3
+fosslight_util>=1.3.4
+fosslight_dependency>=3.7.4

src/fosslight_scanner/_get_input.py

@@ -9,22 +9,13 @@
 
 
 def get_input(ask_msg, default_value=""):
-    i = 0
-    result = True
-    for i in range(2):
-        return_value = raw_input(
-            ask_msg) if _PYTHON_VERSION < 3 else input(ask_msg)
-        if return_value != "":
-            break
-        elif default_value != "":
-            break
-        else:
-            i += 1
-    if return_value == "" and default_value == "":
-        result = False
-    elif return_value == "" and default_value != "":
+
+    return_value = raw_input(
+        ask_msg) if _PYTHON_VERSION < 3 else input(ask_msg)
+
+    if return_value == "" and default_value != "":
         return_value = default_value
-    return result, return_value
+    return return_value
 
 
 def ask_to_run(ask_msg):
@@ -38,23 +29,20 @@ def get_input_mode():
     global _PYTHON_VERSION
     _PYTHON_VERSION = sys.version_info[0]
 
-    _input_path_msg = "Please enter the path to analyze:"
-
     url_to_analyze = ""
     src_path = ""
     dep_path = ""
     dep_arguments = ""
 
-    # 0. Select to analyze
-    if ask_to_run("0. What are you going to analyze? (1/2)\n\
-        \t1. Links that can be cloned by git or wget\n\
-            2. Local source path\n"):
-        success, url_to_analyze = get_input("Enter the link to analyze:", "")
+    if ask_to_run("What are you going to analyze? (1/2)\n\
+                   1. Links that can be cloned by git or wget\n\
+                   2. Local source path\n"):
+        url_to_analyze = get_input("Enter the link to analyze:", "")
     else:
-        success, src_path = get_input(_input_path_msg, "")
+        src_path = get_input("Please enter the path to analyze:", "")
         dep_path = src_path
 
-        success, dep_arguments = get_input(
+        dep_arguments = get_input(
             "Please enter arguments for dependency analysis:", "")
 
     return src_path, dep_path, dep_arguments, url_to_analyze

src/fosslight_scanner/_help.py

@@ -18,7 +18,8 @@
         -p <source_path>\t\t\t    Path to analyze source, dependency
         -w <link>\t\t\t\t    Link to be analyzaed can be downloaded by wget or git clone
         -o <output>\t\t\t\t    Output directory or file
-        -d <dependency_argument> \t\t\t    Additional arguments for running dependency analysis"""
+        -f <format>\t\t\t\t    Output file format (excel, csv, opossum)
+        -d <dependency_argument> \t\t    Additional arguments for running dependency analysis"""
 
 
 def print_help_msg():

src/fosslight_scanner/fosslight_scanner.py

@@ -14,113 +14,167 @@
 from shutil import rmtree as rmdir
 from datetime import datetime
 from fosslight_source import run_scancode
-from fosslight_dependency.run_dependency_scanner import main as dep_main
+from fosslight_dependency.run_dependency_scanner import run_dependency_scanner
 from fosslight_util.download import cli_download_and_extract
 from ._get_input import get_input_mode
 from ._help import print_help_msg
 from fosslight_util.set_log import init_log
-from fosslight_util.write_excel import merge_excels
 from fosslight_util.timer_thread import TimerThread
 import fosslight_util.constant as constant
+from fosslight_util.output_format import write_output_file, check_output_format
 
-_OUTPUT_FILE_PREFIX = "FOSSLight-Report_"
-_PKG_NAME = "fosslight_scanner"
+OUTPUT_EXCEL_PREFIX = "FOSSLight-Report_"
+OUTPUT_JSON_PREFIX = "Opossum_input_"
+PKG_NAME = "fosslight_scanner"
 logger = logging.getLogger(constant.LOGGER_NAME)
 warnings.simplefilter(action='ignore', category=FutureWarning)
 _output_dir = "fosslight_raw_data_"
 _log_file = "fosslight_log_"
 _start_time = ""
 _executed_path = ""
-_SRC_DIR_FROM_LINK_PREFIX = "fosslight_src_dir_"
-OUTPUT_FILE_EXTENSION = ".xlsx"
+SRC_DIR_FROM_LINK_PREFIX = "fosslight_src_dir_"
 
 
 def run_analysis(path_to_run, params, func, str_run_start, output, exe_path):
     # This function will be replaced by call_analysis_api().
     logger.info("## Start to run "+str_run_start)
+    return_value = ""
     try:
         if path_to_run != "":
-            logger.info("|--- Path to analyze :"+path_to_run)
+            logger.info("|--- Path to analyze :" + path_to_run)
             os.chdir(output)
             sys.argv = params
-            func()
+            return_value = func()
             os.chdir(exe_path)
         else:
             logger.info("Analyzing path is missing...")
     except SystemExit:
         pass
     except Exception as ex:
         logger.error(str_run_start + ":" + str(ex))
+    return return_value
 
 
-def call_analysis_api(path_to_run, str_run_start, func, *args):
+def call_analysis_api(path_to_run, str_run_start, return_idx, func, *args):
+    # return_idx == -1 : Raw return value itself
     logger.info("## Start to run " + str_run_start)
+    success = True
+    result = []
     try:
         if path_to_run != "":
             logger.info("|--- Path to analyze :"+path_to_run)
-            success, msg, result_list = func(*args)
-            if not success:
-                logger.warning(msg)
+            result = func(*args)
         else:
             logger.info("Analyzing path is missing...")
     except SystemExit:
-        pass
+        success = False
     except Exception as ex:
+        success = False
         logger.error(str_run_start + ":" + str(ex))
+    try:
+        if success:
+            if result and return_idx >= 0:
+                if len(result) > return_idx:
+                    result = result[return_idx]
+                else:
+                    success = False
+    except Exception as ex:
+        logger.debug("Get return value:" + str(ex))
+        success = False
+    if not result:
+        result = []
+    return success, result
+
 
+def run_dependency(path_to_analyze, output_file_with_path, params=""):
+    result_list = []
+
+    package_manager = ""
+    pip_activate_cmd = ""
+    pip_deactivate_cmd = ""
+    output_custom_dir = ""
+    app_name = ""
+    github_token = ""
 
-def set_sub_parameter(default_params, params):  # For dependency
     try:
         if params != "":
             match_obj = re.findall(
-                r'\s*(-\s*[a|d|m|c|p|v|o])\s*\'([^\']+)\'\s*', params)
+                r'\s*(-\s*[a|d|m|c|n|t])\s*\'([^\']+)\'\s*', params)
             for param, value in match_obj:
-                default_params.append(param)
-                default_params.append(value)
+                if param == "-m":
+                    package_manager = value
+                elif param == "-a":
+                    pip_activate_cmd = value
+                elif param == "-d":
+                    pip_deactivate_cmd = value
+                elif param == "-c":
+                    output_custom_dir = value
+                elif param == "-n":
+                    app_name = value
+                elif param == "-t":
+                    github_token = value
+    except Exception as ex:
+        logger.warning("Set dependency Param:" + str(ex))
+
+    try:
+        success, result = call_analysis_api(path_to_analyze, "Dependency Analysis",
+                                            1, run_dependency_scanner,
+                                            package_manager,
+                                            os.path.abspath(path_to_analyze),
+                                            output_file_with_path,
+                                            pip_activate_cmd, pip_deactivate_cmd,
+                                            output_custom_dir, app_name,
+                                            github_token)
+        if success:
+            result_list = result.get('SRC_FL_Dependency')
     except Exception as ex:
-        logger.warning("SET dependency Param:"+str(ex))
-    return default_params
+        logger.warning("Run dependency:"+str(ex))
+
+    if not result_list:
+        result_list = []
+    return result_list
 
 
 def run(src_path, dep_path, dep_arguments, output_path, remove_raw_data=True,
-        remove_src_data=True, need_init=True, result_log={}, output_file=""):
+        remove_src_data=True, need_init=True, result_log={}, output_file="", output_extension=""):
     try:
         success = True
+        sheet_list = {}
         if need_init:
             success, final_excel_dir, result_log = init(output_path)
         else:
             final_excel_dir = output_path
+        final_excel_dir = os.path.abspath(final_excel_dir)
+
         if output_file == "":
-            output_file = _OUTPUT_FILE_PREFIX + _start_time + OUTPUT_FILE_EXTENSION
+            output_prefix = OUTPUT_EXCEL_PREFIX if output_extension != ".json" else OUTPUT_JSON_PREFIX
+            output_file = output_prefix + _start_time
 
         if success:
             output_files = {"SRC": "FL_Source",
                             "BIN": "FL_Binary.xlsx",
                             "DEP": "FL_Dependency.xlsx",
-                            "REUSE": "reuse.xml",
-                            "FINAL": output_file}
-            if src_path != "":
-                call_analysis_api(src_path, "Source Analysis",
-                                  run_scancode.run_scan,
-                                  os.path.abspath(src_path),
-                                  os.path.join(_output_dir, output_files["SRC"]),
-                                  False)
-            if dep_path != "":
-                run_analysis(dep_path,
-                             set_sub_parameter(["DEP",
-                                                "-p", os.path.abspath(dep_path),
-                                                "-o", os.path.join(_output_dir, output_files["DEP"])],
-                                               dep_arguments),
-                             dep_main, "Dependency Analysis",
-                             _output_dir, _executed_path)
-
-            ouput_file = os.path.join(final_excel_dir, output_files["FINAL"])
-            success, error_msg = merge_excels(_output_dir, ouput_file)
+                            "REUSE": "reuse.xml"}
+
+            success, result = call_analysis_api(src_path, "Source Analysis",
+                                                2, run_scancode.run_scan,
+                                                os.path.abspath(src_path),
+                                                os.path.join(_output_dir, output_files["SRC"]),
+                                                False, -1, True)
+            if success:
+                sheet_list["SRC_FL_Source"] = [scan_item.get_row_to_print() for scan_item in result]
+
+            result_list = run_dependency(dep_path, os.path.join(_output_dir, output_files["DEP"]), dep_arguments)
+            sheet_list['SRC_FL_Dependency'] = result_list
+
+            output_file_without_ext = os.path.join(final_excel_dir, output_file)
+            success, msg = write_output_file(output_file_without_ext, output_extension, sheet_list)
+
             result_log["Result"] = success
             if success:
-                result_log["FOSSLight-Report"] = ouput_file
+                result_log["Output Path"] = final_excel_dir
             else:
-                result_log["Result Message - Merge"] = error_msg
+                result_log["Result Message - Merge"] = msg
     except Exception as ex:
         logger.error("Scanning:" + str(ex))
 
@@ -141,12 +195,12 @@ def run(src_path, dep_path, dep_arguments, output_path, remove_raw_data=True,
         logger.debug("Error to remove temp files:"+str(ex))
 
 
-def run_after_download_source(link, out_dir, remove_raw_data, output_file=""):
+def run_after_download_source(link, out_dir, remove_raw_data, output_file="", output_extension=""):
     start_time = datetime.now().strftime('%Y%m%d_%H%M%S')
     try:
         success, final_excel_dir, result_log = init(out_dir)
         temp_src_dir = os.path.join(
-            _output_dir, _SRC_DIR_FROM_LINK_PREFIX+start_time)
+            _output_dir, SRC_DIR_FROM_LINK_PREFIX+start_time)
 
         logger.info("Link to download :"+link)
         success, msg = cli_download_and_extract(
@@ -156,7 +210,7 @@ def run_after_download_source(link, out_dir, remove_raw_data, output_file=""):
             logger.info("Downloaded Dir:"+temp_src_dir)
             run(temp_src_dir, temp_src_dir,
                 "", final_excel_dir, remove_raw_data, remove_raw_data, False,
-                result_log, output_file)
+                result_log, output_file, output_extension)
         else:
             logger.error("Download failed:" + msg)
     except Exception as ex:
@@ -184,7 +238,7 @@ def init(output_path=""):
 
     log_dir = os.path.join(output_root_dir, "fosslight_log")
     logger, result_log = init_log(os.path.join(log_dir, _log_file + _start_time + ".txt"),
-                                  True, logging.INFO, logging.DEBUG, _PKG_NAME)
+                                  True, logging.INFO, logging.DEBUG, PKG_NAME)
 
     return os.path.isdir(_output_dir), output_root_dir, result_log
 
@@ -200,14 +254,15 @@ def main():
     url_to_analyze = ""
     _executed_path = os.getcwd()
     remove_raw_data = True
-    output_dir = _executed_path
+    output_path = _executed_path
     output_file = ""
     output_file_or_dir = ""
     show_progressbar = True
+    file_format = ""
 
     try:
         argv = sys.argv[1:]
-        opts, args = getopt.getopt(argv, 'htrs:d:a:o:w:')
+        opts, args = getopt.getopt(argv, 'htrs:d:a:o:w:f:p:')
     except getopt.GetoptError:
         print_help_msg()
 
@@ -229,14 +284,14 @@ def main():
             remove_raw_data = False
         elif opt == "-t":
             show_progressbar = False
+        elif opt == "-f":
+            file_format = arg
 
     try:
-        if output_file_or_dir != "":
-            if output_file_or_dir.endswith(OUTPUT_FILE_EXTENSION):
-                output_file = Path(output_file_or_dir).name
-                output_dir = os.path.dirname(os.path.abspath(output_file_or_dir))
-            else:
-                output_dir = os.path.abspath(output_file_or_dir)
+        success, msg, output_path, output_file, output_extension = check_output_format(output_file_or_dir, file_format)
+        if not success:
+            logger.error(msg)
+            return False
 
         if not _cli_mode:
             src_path, dep_path, dep_arguments, url_to_analyze = get_input_mode()
@@ -246,10 +301,10 @@ def main():
             timer.start()
 
         if url_to_analyze != "":
-            run_after_download_source(url_to_analyze, output_dir, remove_raw_data, output_file)
+            run_after_download_source(url_to_analyze, output_path, remove_raw_data, output_file, output_extension)
         elif src_path != "" or dep_path != "":
             run(src_path, dep_path,
-                dep_arguments, output_dir, remove_raw_data, False, True, {}, output_file)
+                dep_arguments, output_path, remove_raw_data, False, True, {}, output_file, output_extension)
     except Exception as ex:
         logger.warning(str(ex))