Download the latest version of package url

Signed-off-by: Jiyeong Seok <[email protected]>

Author: dd-jy

requirements.txt

@@ -9,3 +9,5 @@ fosslight_source>=1.7.3
 fosslight_dependency>=3.7.4
 fosslight_binary>=4.1.24
 fosslight_prechecker>=3.0.1
+npm
+requests

src/fosslight_scanner/common.py

@@ -11,6 +11,9 @@
 import pandas as pd
 import yaml
 import copy
+import requests
+from npm.bindings import npm_run
+from lastversion import latest
 import fosslight_util.constant as constant
 from fosslight_util.parsing_yaml import parsing_yml
 from fosslight_util.write_yaml import create_yaml_with_ossitem
@@ -90,61 +93,92 @@ def call_analysis_api(path_to_run, str_run_start, return_idx, func, *args):
     return success, result
 
 
-def extract_name_from_link(link):
+def extract_name_version_from_link(link):
     # Github : https://github.com/(owner)/(repo)
-    # npm : www.npmjs.com/package/(package)
-    # npm : https://www.npmjs.com/package/@(group)/(package)
-    # pypi : https://pypi.org/project/(oss_name)
+    # npm : https://www.npmjs.com/package/(package)/v/(version)
+    # npm2 : https://www.npmjs.com/package/@(group)/(package)/v/(version)
+    # pypi : https://pypi.org/project/(oss_name)/(version)
     # pypi2 : https://files.pythonhosted.org/packages/source/(alphabet)/(oss_name)/(oss_name)-(version).tar.gz
-    # Maven: https://mvnrepository.com/artifact/(group)/(artifact)
-    # pub: https://pub.dev/packages/(package)
+    # Maven: https://mvnrepository.com/artifact/(group)/(artifact)/(version)
+    # pub: https://pub.dev/packages/(package)/versions/(version)
     # Cocoapods: https://cocoapods.org/(package)
     pkg_pattern = {
         "github": r'https?:\/\/github.com\/([^\/]+)\/([^\/\.]+)(\.git)?',
-        "pypi": r'https?:\/\/pypi\.org\/project\/([^\/]+)',
-        "pypi2": r'https?:\/\/files\.pythonhosted\.org\/packages\/source\/[\w]\/([^\/]+)\/([^\/]+)',
-        "maven": r'https?:\/\/mvnrepository\.com\/artifact\/([^\/]+)\/([^\/]+)',
-        "npm": r'https?:\/\/www\.npmjs\.com\/package\/([^\/]+)(\/[^\/]+)?',
-        "pub": r'https?:\/\/pub\.dev\/packages\/([^\/]+)',
+        "pypi": r'https?:\/\/pypi\.org\/project\/([^\/]+)[\/]?([^\/]*)',
+        "pypi2": r'https?:\/\/files\.pythonhosted\.org\/packages\/source\/[\w]\/([^\/]+)\/[\S]+-([^\-]+)\.tar\.gz',
+        "maven": r'https?:\/\/mvnrepository\.com\/artifact\/([^\/]+)\/([^\/]+)\/?([^\/]*)',
+        "npm": r'https?:\/\/www\.npmjs\.com\/package\/([^\/\@]+)(?:\/v\/)?([^\/]*)',
+        "npm2": r'https?:\/\/www\.npmjs\.com\/package\/(\@[^\/]+\/[^\/]+)(?:\/v\/)?([^\/]*)',
+        "pub": r'https?:\/\/pub\.dev\/packages\/([^\/]+)(?:\/versions\/)?([^\/]*)',
         "pods": r'https?:\/\/cocoapods\.org\/pods\/([^\/]+)'
     }
     oss_name = ""
+    oss_version = ""
     if link.startswith("www."):
         link = link.replace("www.", "https://www.", 1)
     for key, value in pkg_pattern.items():
-        try:
-            p = re.compile(value)
-            match = p.match(link)
-            if match:
-                group = match.group(1)
+        p = re.compile(value)
+        match = p.match(link)
+        if match:
+            try:
+                origin_name = match.group(1)
                 if key == "github":
                     repo = match.group(2)
-                    oss_name = f"{group}-{repo}"
-                    break
+                    oss_name = f"{origin_name}-{repo}"
                 elif (key == "pypi") or (key == "pypi2"):
-                    oss_name = f"pypi:{group}"
+                    oss_name = f"pypi:{origin_name}"
                     oss_name = re.sub(r"[-_.]+", "-", oss_name).lower()
-                    break
+                    oss_version = match.group(2)
                 elif key == "maven":
                     artifact = match.group(2)
-                    oss_name = f"{group}:{artifact}"
-                    break
-                elif key == "npm":
-                    if group.startswith("@"):
-                        pkg = match.group(2)
-                        oss_name = f"npm:{group}{pkg}"
-                    else:
-                        oss_name = f"npm:{group}"
-                    break
+                    oss_name = f"{origin_name}:{artifact}"
+                    origin_name = oss_name
+                    oss_version = match.group(3)
+                elif key == "npm" or key == "npm2":
+                    oss_name = f"npm:{origin_name}"
+                    oss_version = match.group(2)
                 elif key == "pub":
-                    oss_name = f"pub:{group}"
-                    break
+                    oss_name = f"pub:{origin_name}"
+                    oss_version = match.group(2)
                 elif key == "pods":
-                    oss_name = f"cocoapods:{group}"
-                    break
-        except Exception as ex:
-            logger.debug(f"extract_name_from_link_{key}:{ex}")
-    return oss_name
+                    oss_name = f"cocoapods:{origin_name}"
+            except Exception as ex:
+                logger.info(f"extract_name_version_from_link {key}:{ex}")
+            if oss_name and (not oss_version):
+                if key in ["pypi", "maven", "npm", "npm2", "pub"]:
+                    oss_version, link = get_latest_package_version(link, key, origin_name)
+                    logger.debug(f'Try to download with the latest version:{link}')
+            break
+    return oss_name, oss_version, link
+
+
+def get_latest_package_version(link, pkg_type, oss_name):
+    find_version = ''
+    link_with_version = link
+
+    try:
+        if pkg_type in ['npm', 'npm2']:
+            stderr, stdout = npm_run('view', oss_name, 'version')
+            if stdout:
+                find_version = stdout.strip()
+            link_with_version = f'https://www.npmjs.com/package/{oss_name}/v/{find_version}'
+        elif pkg_type == 'pypi':
+            find_version = str(latest(oss_name, at='pip', output_format='version', pre_ok=True))
+            link_with_version = f'https://pypi.org/project/{oss_name}/{find_version}'
+        elif pkg_type == 'maven':
+            maven_response = requests.get(f'https://api.deps.dev/v3alpha/systems/maven/packages/{oss_name}')
+            if maven_response.status_code == 200:
+                find_version = maven_response.json().get('versions')[-1].get('versionKey').get('version')
+            oss_name = oss_name.replace(':', '/')
+            link_with_version = f'https://mvnrepository.com/artifact/{oss_name}/{find_version}'
+        elif pkg_type == 'pub':
+            pub_response = requests.get(f'https://pub.dev/api/packages/{oss_name}')
+            if pub_response.status_code == 200:
+                find_version = pub_response.json().get('latest').get('version')
+            link_with_version = f'https://pub.dev/packages/{oss_name}/versions/{find_version}'
+    except Exception as e:
+        logger.debug(f'Fail to get latest package version({link}:{e})')
+    return find_version, link_with_version
 
 
 def overwrite_excel(excel_file_path, oss_name, column_name='OSS Name'):
@@ -169,7 +203,8 @@ def overwrite_excel(excel_file_path, oss_name, column_name='OSS Name'):
             logger.debug(f"overwrite_excel:{ex}")
 
 
-def merge_yamls(_output_dir, merge_yaml_files, final_report, remove_src_data=False, default_oss_name='', url=''):
+def merge_yamls(_output_dir, merge_yaml_files, final_report, remove_src_data=False,
+                default_oss_name='', default_oss_version='', url=''):
     success = True
     err_msg = ''
 
@@ -183,7 +218,8 @@ def merge_yamls(_output_dir, merge_yaml_files, final_report, remove_src_data=Fal
                 if remove_src_data:
                     existed_yaml = {}
                     for oi in oss_list:
-                        oi.name = default_oss_name if oi.name == '-' else oi.name
+                        oi.name = default_oss_name if oi.name == '' else oi.name
+                        oi.version = default_oss_version if oi.version == '' else oi.version
                         oi.download_location = url if oi.download_location == '' else oi.download_location
                         create_yaml_with_ossitem(oi, existed_yaml)
                     with open(os.path.join(_output_dir, mf), 'w') as f:

src/fosslight_scanner/fosslight_scanner.py

@@ -23,7 +23,7 @@
 from fosslight_util.output_format import check_output_format
 from fosslight_prechecker._precheck import run_lint as prechecker_lint
 from .common import (copy_file, call_analysis_api,
-                     overwrite_excel, extract_name_from_link,
+                     overwrite_excel, extract_name_version_from_link,
                      merge_yamls, correct_scanner_result,
                      create_scancodejson)
 from fosslight_util.write_excel import merge_excels
@@ -104,7 +104,7 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
                 run_src=True, run_bin=True, run_dep=True, run_prechecker=True,
                 remove_src_data=True, result_log={}, output_file="",
                 output_extension="", num_cores=-1, db_url="",
-                default_oss_name="", url="",
+                default_oss_name="", default_oss_version="", url="",
                 correct_mode=True, correct_fpath="", ui_mode=False):
     final_excel_dir = output_path
     success = True
@@ -206,6 +206,7 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
 
             if remove_src_data:
                 overwrite_excel(_output_dir, default_oss_name, "OSS Name")
+                overwrite_excel(_output_dir, default_oss_version, "OSS Version")
                 overwrite_excel(_output_dir, url, "Download Location")
             success, err_msg = merge_excels(_output_dir, final_report, merge_files)
 
@@ -219,7 +220,7 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
                 shutil.rmtree(os.path.join(_output_dir, tmp_dir), ignore_errors=True)
         elif output_extension == ".yaml":
             success, err_msg = merge_yamls(_output_dir, merge_files, final_report,
-                                           remove_src_data, default_oss_name, url)
+                                           remove_src_data, default_oss_name, default_oss_version, url)
         if success:
             if os.path.isfile(final_report):
                 logger.info(f'Generated the result file: {final_report}')
@@ -305,6 +306,7 @@ def run_main(mode, path_arg, dep_arguments, output_file_or_dir, file_format, url
 
     output_file = ""
     default_oss_name = ""
+    default_oss_version = ""
     src_path = ""
     _executed_path = os.getcwd()
 
@@ -392,7 +394,7 @@ def run_main(mode, path_arg, dep_arguments, output_file_or_dir, file_format, url
 
                 if url_to_analyze != "":
                     remove_downloaded_source = True
-                    default_oss_name = extract_name_from_link(url_to_analyze)
+                    default_oss_name, default_oss_version, url_to_analyze = extract_name_version_from_link(url_to_analyze)
                     success, src_path = download_source(url_to_analyze, output_path)
 
                 if output_extension == ".yaml":
@@ -407,7 +409,7 @@ def run_main(mode, path_arg, dep_arguments, output_file_or_dir, file_format, url
                                 run_src, run_bin, run_dep, run_prechecker,
                                 remove_downloaded_source, {}, output_file,
                                 output_extension, num_cores, db_url,
-                                default_oss_name, url_to_analyze,
+                                default_oss_name, default_oss_version, url_to_analyze,
                                 correct_mode, correct_fpath, ui_mode)
             else:
                 logger.error("No mode has been selected for analysis.")