Fix the pub download url bug (#147)

* Fix the pub download url bug

Signed-off-by: Jiyeong Seok <[email protected]>

* Add to get oss info with package url

Signed-off-by: Jiyeong Seok <[email protected]>

---------

Signed-off-by: Jiyeong Seok <[email protected]>

Author: dd-jy

requirements.txt

@@ -10,7 +10,8 @@ python3-wget
 beautifulsoup4
 jsonmerge
 spdx-tools==0.7.0rc0
-npm
 setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability
 numpy; python_version < '3.8'
-numpy>=1.22.2; python_version >= '3.8'
+numpy>=1.22.2; python_version >= '3.8'
+npm
+requests

src/fosslight_util/_get_downloadable_url.py

@@ -4,32 +4,119 @@
 # SPDX-License-Identifier: Apache-2.0
 import logging
 import re
+import requests
+from npm.bindings import npm_run
+from lastversion import latest
 from bs4 import BeautifulSoup
 from urllib.request import urlopen
 import fosslight_util.constant as constant
-from npm.bindings import npm_run
 
 logger = logging.getLogger(constant.LOGGER_NAME)
 
 
+def extract_name_version_from_link(link):
+    # Github : https://github.com/(owner)/(repo)
+    # npm : https://www.npmjs.com/package/(package)/v/(version)
+    # npm2 : https://www.npmjs.com/package/@(group)/(package)/v/(version)
+    # pypi : https://pypi.org/project/(oss_name)/(version)
+    # pypi2 : https://files.pythonhosted.org/packages/source/(alphabet)/(oss_name)/(oss_name)-(version).tar.gz
+    # Maven: https://mvnrepository.com/artifact/(group)/(artifact)/(version)
+    # pub: https://pub.dev/packages/(package)/versions/(version)
+    # Cocoapods: https://cocoapods.org/(package)
+    pkg_pattern = {
+        "pypi": r'https?:\/\/pypi\.org\/project\/([^\/]+)[\/]?([^\/]*)',
+        "pypi2": r'https?:\/\/files\.pythonhosted\.org\/packages\/source\/[\w]\/([^\/]+)\/[\S]+-([^\-]+)\.tar\.gz',
+        "maven": r'https?:\/\/mvnrepository\.com\/artifact\/([^\/]+)\/([^\/]+)\/?([^\/]*)',
+        "npm": r'https?:\/\/www\.npmjs\.com\/package\/([^\/\@]+)(?:\/v\/)?([^\/]*)',
+        "npm2": r'https?:\/\/www\.npmjs\.com\/package\/(\@[^\/]+\/[^\/]+)(?:\/v\/)?([^\/]*)',
+        "pub": r'https?:\/\/pub\.dev\/packages\/([^\/]+)(?:\/versions\/)?([^\/]*)',
+        "pods": r'https?:\/\/cocoapods\.org\/pods\/([^\/]+)'
+    }
+    oss_name = ""
+    oss_version = ""
+    if link.startswith("www."):
+        link = link.replace("www.", "https://www.", 1)
+    for key, value in pkg_pattern.items():
+        p = re.compile(value)
+        match = p.match(link)
+        if match:
+            try:
+                origin_name = match.group(1)
+                if (key == "pypi") or (key == "pypi2"):
+                    oss_name = f"pypi:{origin_name}"
+                    oss_name = re.sub(r"[-_.]+", "-", oss_name).lower()
+                    oss_version = match.group(2)
+                elif key == "maven":
+                    artifact = match.group(2)
+                    oss_name = f"{origin_name}:{artifact}"
+                    origin_name = oss_name
+                    oss_version = match.group(3)
+                elif key == "npm" or key == "npm2":
+                    oss_name = f"npm:{origin_name}"
+                    oss_version = match.group(2)
+                elif key == "pub":
+                    oss_name = f"pub:{origin_name}"
+                    oss_version = match.group(2)
+                elif key == "pods":
+                    oss_name = f"cocoapods:{origin_name}"
+            except Exception as ex:
+                logger.info(f"extract_name_version_from_link {key}:{ex}")
+            if oss_name and (not oss_version):
+                if key in ["pypi", "maven", "npm", "npm2", "pub"]:
+                    oss_version, link = get_latest_package_version(link, key, origin_name)
+                    logger.debug(f'Try to download with the latest version:{link}')
+            break
+    return oss_name, oss_version, link, key
+
+
+def get_latest_package_version(link, pkg_type, oss_name):
+    find_version = ''
+    link_with_version = link
+
+    try:
+        if pkg_type in ['npm', 'npm2']:
+            stderr, stdout = npm_run('view', oss_name, 'version')
+            if stdout:
+                find_version = stdout.strip()
+            link_with_version = f'https://www.npmjs.com/package/{oss_name}/v/{find_version}'
+        elif pkg_type == 'pypi':
+            find_version = str(latest(oss_name, at='pip', output_format='version', pre_ok=True))
+            link_with_version = f'https://pypi.org/project/{oss_name}/{find_version}'
+        elif pkg_type == 'maven':
+            maven_response = requests.get(f'https://api.deps.dev/v3alpha/systems/maven/packages/{oss_name}')
+            if maven_response.status_code == 200:
+                find_version = maven_response.json().get('versions')[-1].get('versionKey').get('version')
+            oss_name = oss_name.replace(':', '/')
+            link_with_version = f'https://mvnrepository.com/artifact/{oss_name}/{find_version}'
+        elif pkg_type == 'pub':
+            pub_response = requests.get(f'https://pub.dev/api/packages/{oss_name}')
+            if pub_response.status_code == 200:
+                find_version = pub_response.json().get('latest').get('version')
+            link_with_version = f'https://pub.dev/packages/{oss_name}/versions/{find_version}'
+    except Exception as e:
+        logger.debug(f'Fail to get latest package version({link}:{e})')
+    return find_version, link_with_version
+
+
 def get_downloadable_url(link):
 
     ret = False
-    new_link = ''
+    result_link = link
 
-    link = link.replace('http://', '')
-    link = link.replace('https://', '')
+    oss_name, oss_version, new_link, pkg_type = extract_name_version_from_link(link)
+    new_link = new_link.replace('http://', '')
+    new_link = new_link.replace('https://', '')
 
-    if link.startswith('pypi.org/'):
-        ret, new_link = get_download_location_for_pypi(link)
-    elif link.startswith('mvnrepository.com/artifact/') or link.startswith('repo1.maven.org/'):
-        ret, new_link = get_download_location_for_maven(link)
-    elif link.startswith('www.npmjs.com/') or link.startswith('registry.npmjs.org/'):
-        ret, new_link = get_download_location_for_npm(link)
-    elif link.startswith('pub.dev/'):
-        ret, new_link = get_download_location_for_pub(link)
+    if pkg_type == "pypi":
+        ret, result_link = get_download_location_for_pypi(new_link)
+    elif pkg_type == "maven" or new_link.startswith('repo1.maven.org/'):
+        ret, result_link = get_download_location_for_maven(new_link)
+    elif (pkg_type in ["npm", "npm2"]) or new_link.startswith('registry.npmjs.org/'):
+        ret, result_link = get_download_location_for_npm(new_link)
+    elif pkg_type == "pub":
+        ret, result_link = get_download_location_for_pub(new_link)
 
-    return ret, new_link
+    return ret, result_link, oss_name, oss_version
 
 
 def get_download_location_for_pypi(link):
@@ -134,16 +221,9 @@ def get_download_location_for_npm(link):
                 oss_name_npm = dn_loc_split[idx]
                 tar_name = oss_name_npm
                 oss_version = dn_loc_split[idx+2]
-        except Exception:
-            pass
 
-        try:
-            if not oss_version:
-                stderr, stdout = npm_run('view', oss_name_npm, 'version')
-                if stdout:
-                    oss_version = stdout.strip()
-            tar_name = f"{tar_name}-{oss_version}"
-            new_link = 'https://registry.npmjs.org/' + oss_name_npm + '/-/' + tar_name + '.tgz'
+            tar_name = f'{tar_name}-{oss_version}'
+            new_link = f'https://registry.npmjs.org/{oss_name_npm}/-/{tar_name}.tgz'
             ret = True
         except Exception as error:
             ret = False
@@ -159,7 +239,7 @@ def get_download_location_for_pub(link):
     # download url format : https://pub.dev/packages/(oss_name)/versions/(oss_version).tar.gz
     try:
         if link.startswith('pub.dev/packages'):
-            new_link = 'https://{link}.tar.gz'
+            new_link = f'https://{link}.tar.gz'
             ret = True
 
     except Exception as error:

src/fosslight_util/download.py

@@ -24,6 +24,7 @@
 import threading
 import platform
 import subprocess
+import re
 
 logger = logging.getLogger(constant.LOGGER_NAME)
 compression_extension = {".tar.bz2", ".tar.gz", ".tar.xz", ".tgz", ".tar", ".zip", ".jar", ".bz2"}
@@ -115,6 +116,8 @@ def cli_download_and_extract(link, target_dir, log_dir, checkout_to="", compress
 
     success = True
     msg = ""
+    oss_name = ""
+    oss_version = ""
     log_file_name = "fosslight_download_" + \
         datetime.now().strftime('%Y%m%d_%H-%M-%S')+".txt"
     logger, log_item = init_log(os.path.join(log_dir, log_file_name))
@@ -135,22 +138,29 @@ def cli_download_and_extract(link, target_dir, log_dir, checkout_to="", compress
             is_rubygems = src_info.get("rubygems", False)
 
             # General download (git clone, wget)
-            if (not is_rubygems) and (not download_git_clone(link, target_dir, checkout_to, tag, branch)):
+            success_git, msg, oss_name = download_git_clone(link, target_dir, checkout_to, tag, branch)
+            if (not is_rubygems) and (not success_git):
                 if os.path.isfile(target_dir):
                     shutil.rmtree(target_dir)
 
-                success, downloaded_file = download_wget(link, target_dir, compressed_only)
+                success, downloaded_file, msg_wget, oss_name, oss_version = download_wget(link, target_dir, compressed_only)
                 if success:
                     success = extract_compressed_file(downloaded_file, target_dir, True)
             # Download from rubygems.org
             elif is_rubygems and shutil.which("gem"):
                 success = gem_download(link, target_dir, checkout_to)
+        if msg:
+            msg = f'git fail: {msg}'
+            if msg_wget:
+                msg = f'{msg}, wget fail: {msg_wget}'
+            else:
+                msg = f'{msg}, wget success'
     except Exception as error:
         success = False
         msg = str(error)
 
-    logger.info(f"\n* FOSSLight Downloader - Result: {success}\n {msg}")
-    return success, msg
+    logger.info(f"\n* FOSSLight Downloader - Result: {success} ({msg})")
+    return success, msg, oss_name, oss_version
 
 
 def get_ref_to_checkout(checkout_to, ref_list):
@@ -184,8 +194,19 @@ def decide_checkout(checkout_to="", tag="", branch=""):
     return ref_to_checkout
 
 
+def get_github_ossname(link):
+    oss_name = ""
+    p = re.compile(r'https?:\/\/github.com\/([^\/]+)\/([^\/\.]+)(\.git)?')
+    match = p.match(link)
+    if match:
+        oss_name = f"{match.group(1)}-{match.group(2)}"
+    return oss_name
+
+
 def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch=""):
     ref_to_checkout = decide_checkout(checkout_to, tag, branch)
+    msg = ""
+    oss_name = get_github_ossname(git_url)
 
     if platform.system() != "Windows":
         signal.signal(signal.SIGALRM, alarm_handler)
@@ -204,7 +225,8 @@ def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch=""):
             del alarm
     except Exception as error:
         logger.warning(f"git clone - failed: {error}")
-        return False
+        msg = str(error)
+        return False, msg, oss_name
     try:
         if ref_to_checkout != "":
             ref_list = [x for x in repo.references]
@@ -213,11 +235,14 @@ def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch=""):
             repo.checkout(ref_to_checkout)
     except Exception as error:
         logger.warning(f"git checkout to {ref_to_checkout} - failed: {error}")
-    return True
+    return True, msg, oss_name
 
 
 def download_wget(link, target_dir, compressed_only):
     success = False
+    msg = ""
+    oss_name = ""
+    oss_version = ""
     downloaded_file = ""
     if platform.system() != "Windows":
         signal.signal(signal.SIGALRM, alarm_handler)
@@ -228,7 +253,7 @@ def download_wget(link, target_dir, compressed_only):
     try:
         Path(target_dir).mkdir(parents=True, exist_ok=True)
 
-        ret, new_link = get_downloadable_url(link)
+        ret, new_link, oss_name, oss_version = get_downloadable_url(link)
         if ret and new_link:
             link = new_link
 
@@ -255,9 +280,10 @@ def download_wget(link, target_dir, compressed_only):
             logger.debug(f"wget - downloaded: {downloaded_file}")
     except Exception as error:
         success = False
+        msg = str(error)
         logger.warning(f"wget - failed: {error}")
 
-    return success, downloaded_file
+    return success, downloaded_file, msg, oss_name, oss_version
 
 
 def extract_compressed_dir(src_dir, target_dir, remove_after_extract=True):