Merge pull request #86 from fosslight/latest_version

Download the latest version of package url

Author: dd-jy

requirements.txt

@@ -4,8 +4,8 @@ openpyxl
 progress
 pyyaml
 beautifulsoup4
-fosslight_util>=1.4.32
+fosslight_util>=1.4.36
 fosslight_source>=1.7.3
 fosslight_dependency>=3.7.4
 fosslight_binary>=4.1.24
-fosslight_prechecker>=3.0.1
+fosslight_prechecker>=3.0.1

src/fosslight_scanner/common.py

@@ -7,7 +7,6 @@
 import sys
 import logging
 import shutil
-import re
 import pandas as pd
 import yaml
 import copy
@@ -90,63 +89,6 @@ def call_analysis_api(path_to_run, str_run_start, return_idx, func, *args):
     return success, result
 
 
-def extract_name_from_link(link):
-    # Github : https://github.com/(owner)/(repo)
-    # npm : www.npmjs.com/package/(package)
-    # npm : https://www.npmjs.com/package/@(group)/(package)
-    # pypi : https://pypi.org/project/(oss_name)
-    # pypi2 : https://files.pythonhosted.org/packages/source/(alphabet)/(oss_name)/(oss_name)-(version).tar.gz
-    # Maven: https://mvnrepository.com/artifact/(group)/(artifact)
-    # pub: https://pub.dev/packages/(package)
-    # Cocoapods: https://cocoapods.org/(package)
-    pkg_pattern = {
-        "github": r'https?:\/\/github.com\/([^\/]+)\/([^\/\.]+)(\.git)?',
-        "pypi": r'https?:\/\/pypi\.org\/project\/([^\/]+)',
-        "pypi2": r'https?:\/\/files\.pythonhosted\.org\/packages\/source\/[\w]\/([^\/]+)\/([^\/]+)',
-        "maven": r'https?:\/\/mvnrepository\.com\/artifact\/([^\/]+)\/([^\/]+)',
-        "npm": r'https?:\/\/www\.npmjs\.com\/package\/([^\/]+)(\/[^\/]+)?',
-        "pub": r'https?:\/\/pub\.dev\/packages\/([^\/]+)',
-        "pods": r'https?:\/\/cocoapods\.org\/pods\/([^\/]+)'
-    }
-    oss_name = ""
-    if link.startswith("www."):
-        link = link.replace("www.", "https://www.", 1)
-    for key, value in pkg_pattern.items():
-        try:
-            p = re.compile(value)
-            match = p.match(link)
-            if match:
-                group = match.group(1)
-                if key == "github":
-                    repo = match.group(2)
-                    oss_name = f"{group}-{repo}"
-                    break
-                elif (key == "pypi") or (key == "pypi2"):
-                    oss_name = f"pypi:{group}"
-                    oss_name = re.sub(r"[-_.]+", "-", oss_name).lower()
-                    break
-                elif key == "maven":
-                    artifact = match.group(2)
-                    oss_name = f"{group}:{artifact}"
-                    break
-                elif key == "npm":
-                    if group.startswith("@"):
-                        pkg = match.group(2)
-                        oss_name = f"npm:{group}{pkg}"
-                    else:
-                        oss_name = f"npm:{group}"
-                    break
-                elif key == "pub":
-                    oss_name = f"pub:{group}"
-                    break
-                elif key == "pods":
-                    oss_name = f"cocoapods:{group}"
-                    break
-        except Exception as ex:
-            logger.debug(f"extract_name_from_link_{key}:{ex}")
-    return oss_name
-
-
 def overwrite_excel(excel_file_path, oss_name, column_name='OSS Name'):
     if oss_name != "":
         try:
@@ -169,7 +111,8 @@ def overwrite_excel(excel_file_path, oss_name, column_name='OSS Name'):
             logger.debug(f"overwrite_excel:{ex}")
 
 
-def merge_yamls(_output_dir, merge_yaml_files, final_report, remove_src_data=False, default_oss_name='', url=''):
+def merge_yamls(_output_dir, merge_yaml_files, final_report, remove_src_data=False,
+                default_oss_name='', default_oss_version='', url=''):
     success = True
     err_msg = ''
 
@@ -183,7 +126,8 @@ def merge_yamls(_output_dir, merge_yaml_files, final_report, remove_src_data=Fal
                 if remove_src_data:
                     existed_yaml = {}
                     for oi in oss_list:
-                        oi.name = default_oss_name if oi.name == '-' else oi.name
+                        oi.name = default_oss_name if oi.name == '' else oi.name
+                        oi.version = default_oss_version if oi.version == '' else oi.version
                         oi.download_location = url if oi.download_location == '' else oi.download_location
                         create_yaml_with_ossitem(oi, existed_yaml)
                     with open(os.path.join(_output_dir, mf), 'w') as f:

src/fosslight_scanner/fosslight_scanner.py

@@ -23,7 +23,7 @@
 from fosslight_util.output_format import check_output_format
 from fosslight_prechecker._precheck import run_lint as prechecker_lint
 from .common import (copy_file, call_analysis_api,
-                     overwrite_excel, extract_name_from_link,
+                     overwrite_excel,
                      merge_yamls, correct_scanner_result,
                      create_scancodejson)
 from fosslight_util.write_excel import merge_excels
@@ -104,7 +104,7 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
                 run_src=True, run_bin=True, run_dep=True, run_prechecker=True,
                 remove_src_data=True, result_log={}, output_file="",
                 output_extension="", num_cores=-1, db_url="",
-                default_oss_name="", url="",
+                default_oss_name="", default_oss_version="", url="",
                 correct_mode=True, correct_fpath="", ui_mode=False):
     final_excel_dir = output_path
     success = True
@@ -206,6 +206,7 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
 
             if remove_src_data:
                 overwrite_excel(_output_dir, default_oss_name, "OSS Name")
+                overwrite_excel(_output_dir, default_oss_version, "OSS Version")
                 overwrite_excel(_output_dir, url, "Download Location")
             success, err_msg = merge_excels(_output_dir, final_report, merge_files)
 
@@ -219,7 +220,7 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
                 shutil.rmtree(os.path.join(_output_dir, tmp_dir), ignore_errors=True)
         elif output_extension == ".yaml":
             success, err_msg = merge_yamls(_output_dir, merge_files, final_report,
-                                           remove_src_data, default_oss_name, url)
+                                           remove_src_data, default_oss_name, default_oss_version, url)
         if success:
             if os.path.isfile(final_report):
                 logger.info(f'Generated the result file: {final_report}')
@@ -254,14 +255,16 @@ def download_source(link, out_dir):
     start_time = datetime.now().strftime('%Y%m%d_%H%M%S')
     success = False
     temp_src_dir = ""
+    oss_name = ""
+    oss_version = ""
     try:
         success, final_excel_dir, result_log = init(out_dir)
         temp_src_dir = os.path.join(
             _output_dir, SRC_DIR_FROM_LINK_PREFIX + start_time)
 
         link = link.strip()
         logger.info(f"Link to download: {link}")
-        success, msg = cli_download_and_extract(
+        success, msg, oss_name, oss_version = cli_download_and_extract(
             link, temp_src_dir, _output_dir)
 
         if success:
@@ -272,7 +275,7 @@ def download_source(link, out_dir):
     except Exception as ex:
         success = False
         logger.error(f"Failed to analyze from link: {ex}")
-    return success, temp_src_dir
+    return success, temp_src_dir, oss_name, oss_version
 
 
 def init(output_path="", make_outdir=True):
@@ -305,6 +308,7 @@ def run_main(mode, path_arg, dep_arguments, output_file_or_dir, file_format, url
 
     output_file = ""
     default_oss_name = ""
+    default_oss_version = ""
     src_path = ""
     _executed_path = os.getcwd()
 
@@ -392,8 +396,7 @@ def run_main(mode, path_arg, dep_arguments, output_file_or_dir, file_format, url
 
                 if url_to_analyze != "":
                     remove_downloaded_source = True
-                    default_oss_name = extract_name_from_link(url_to_analyze)
-                    success, src_path = download_source(url_to_analyze, output_path)
+                    success, src_path, default_oss_name, default_oss_version = download_source(url_to_analyze, output_path)
 
                 if output_extension == ".yaml":
                     correct_mode = False
@@ -407,7 +410,7 @@ def run_main(mode, path_arg, dep_arguments, output_file_or_dir, file_format, url
                                 run_src, run_bin, run_dep, run_prechecker,
                                 remove_downloaded_source, {}, output_file,
                                 output_extension, num_cores, db_url,
-                                default_oss_name, url_to_analyze,
+                                default_oss_name, default_oss_version, url_to_analyze,
                                 correct_mode, correct_fpath, ui_mode)
             else:
                 logger.error("No mode has been selected for analysis.")