Support download without ver in url using param

src/fosslight_util/_get_downloadable_url.py

@@ -13,7 +13,7 @@
 logger = logging.getLogger(constant.LOGGER_NAME)
 
 
-def extract_name_version_from_link(link):
+def extract_name_version_from_link(link, checkout_version):
     oss_name = ""
     oss_version = ""
     matched = False
@@ -53,8 +53,12 @@ def extract_name_version_from_link(link):
             except Exception as ex:
                 logger.info(f"extract_name_version_from_link {key}:{ex}")
             if oss_name and (not oss_version):
-                if key in ["pypi", "maven", "npm", "npm2", "pub", "go"]:
-                    oss_version, link = get_latest_package_version(link, key, origin_name)
+                if checkout_version:
+                    oss_version = checkout_version
+                elif key in ["pypi", "maven", "npm", "npm2", "pub", "go"]:
+                    oss_version = get_latest_package_version(link, key, origin_name)
+                if oss_version:
+                    link = get_new_link_with_version(link, key, origin_name, oss_version)
                     logger.info(f'Try to download with the latest version:{link}')
             matched = True
             break
@@ -63,50 +67,56 @@ def extract_name_version_from_link(link):
     return oss_name, oss_version, link, key
 
 
+def get_new_link_with_version(link, pkg_type, oss_name, oss_version):
+    if pkg_type == "pypi":
+        link = f'https://pypi.org/project/{oss_name}/{oss_version}'
+    elif pkg_type == "maven":
+        oss_name = oss_name.replace(':', '/')
+        link = f'https://mvnrepository.com/artifact/{oss_name}/{oss_version}'
+    elif pkg_type == "npm" or pkg_type == "npm2":
+        link = f'https://www.npmjs.com/package/{oss_name}/v/{oss_version}'
+    elif pkg_type == "pub":
+        link = f'https://pub.dev/packages/{oss_name}/versions/{oss_version}'
+    elif pkg_type == "go":
+        link = f'https://pkg.go.dev/{oss_name}@{oss_version}'
+    elif pkg_type == "cargo":
+        link = f'https://crates.io/crates/{oss_name}/{oss_version}'
+    return link
+
+
 def get_latest_package_version(link, pkg_type, oss_name):
     find_version = ''
-    link_with_version = link
 
     try:
         if pkg_type in ['npm', 'npm2']:
             npm_response = requests.get(f"https://registry.npmjs.org/{oss_name}")
             if npm_response.status_code == 200:
                 find_version = npm_response.json().get("dist-tags", {}).get("latest")
-            if find_version:
-                link_with_version = f'https://www.npmjs.com/package/{oss_name}/v/{find_version}'
         elif pkg_type == 'pypi':
             find_version = str(latest(oss_name, at='pip', output_format='version', pre_ok=True))
-            link_with_version = f'https://pypi.org/project/{oss_name}/{find_version}'
         elif pkg_type == 'maven':
             maven_response = requests.get(f'https://api.deps.dev/v3alpha/systems/maven/packages/{oss_name}')
             if maven_response.status_code == 200:
                 find_version = maven_response.json().get('versions')[-1].get('versionKey').get('version')
-            oss_name = oss_name.replace(':', '/')
-            if find_version:
-                link_with_version = f'https://mvnrepository.com/artifact/{oss_name}/{find_version}'
         elif pkg_type == 'pub':
             pub_response = requests.get(f'https://pub.dev/api/packages/{oss_name}')
             if pub_response.status_code == 200:
                 find_version = pub_response.json().get('latest').get('version')
-            if find_version:
-                link_with_version = f'https://pub.dev/packages/{oss_name}/versions/{find_version}'
         elif pkg_type == 'go':
             go_response = requests.get(f'https://proxy.golang.org/{oss_name}/@latest')
             if go_response.status_code == 200:
                 find_version = go_response.json().get('Version')
-            if find_version:
-                link_with_version = f'https://pkg.go.dev/{oss_name}@{find_version}'
     except Exception as e:
         logger.info(f'Fail to get latest package version({link}:{e})')
-    return find_version, link_with_version
+    return find_version
 
 
-def get_downloadable_url(link):
+def get_downloadable_url(link, checkout_version):
 
     ret = False
     result_link = link
 
-    oss_name, oss_version, new_link, pkg_type = extract_name_version_from_link(link)
+    oss_name, oss_version, new_link, pkg_type = extract_name_version_from_link(link, checkout_version)
     new_link = new_link.replace('http://', '')
     new_link = new_link.replace('https://', '')
 

src/fosslight_util/download.py

@@ -137,7 +137,8 @@ def cli_download_and_extract(link: str, target_dir: str, log_dir: str, checkout_
                 if os.path.isfile(target_dir):
                     shutil.rmtree(target_dir)
 
-                success, downloaded_file, msg_wget, oss_name, oss_version = download_wget(link, target_dir, compressed_only)
+                success, downloaded_file, msg_wget, oss_name, oss_version = download_wget(link, target_dir,
+                                                                                          compressed_only, checkout_to)
                 if success:
                     success = extract_compressed_file(downloaded_file, target_dir, True, compressed_only)
             # Download from rubygems.org
@@ -310,7 +311,7 @@ def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch="",
     return success, msg, oss_name, refs_to_checkout
 
 
-def download_wget(link, target_dir, compressed_only):
+def download_wget(link, target_dir, compressed_only, checkout_to):
     success = False
     msg = ""
     oss_name = ""
@@ -327,7 +328,7 @@ def download_wget(link, target_dir, compressed_only):
 
         Path(target_dir).mkdir(parents=True, exist_ok=True)
 
-        ret, new_link, oss_name, oss_version, pkg_type = get_downloadable_url(link)
+        ret, new_link, oss_name, oss_version, pkg_type = get_downloadable_url(link, checkout_to)
         if ret and new_link:
             link = new_link
 

src/fosslight_util/help.py

@@ -34,7 +34,12 @@
     Optional:
         -h\t\t      Print help message
         -t\t\t      Output path name
-        -d\t\t      Directory name to save the log file"""
+        -d\t\t      Directory name to save the log file
+        -s\t\t      Source link to download
+        -t\t\t      Directory to download source code
+        -c\t\t      Checkout to branch or tag/ or version
+        -z\t\t      Unzip only compressed file
+        -o\t\t      Generate summary output file with this option"""
 
 
 class PrintHelpMsg():