Remove the oss name checking with url

Signed-off-by: Jiyeong Seok <[email protected]>

Author: dd-jy

requirements.txt

@@ -8,6 +8,4 @@ fosslight_util>=1.4.32
 fosslight_source>=1.7.3
 fosslight_dependency>=3.7.4
 fosslight_binary>=4.1.24
-fosslight_prechecker>=3.0.1
-npm
-requests
+fosslight_prechecker>=3.0.1

src/fosslight_scanner/common.py

@@ -7,13 +7,9 @@
 import sys
 import logging
 import shutil
-import re
 import pandas as pd
 import yaml
 import copy
-import requests
-from npm.bindings import npm_run
-from lastversion import latest
 import fosslight_util.constant as constant
 from fosslight_util.parsing_yaml import parsing_yml
 from fosslight_util.write_yaml import create_yaml_with_ossitem
@@ -93,94 +89,6 @@ def call_analysis_api(path_to_run, str_run_start, return_idx, func, *args):
     return success, result
 
 
-def extract_name_version_from_link(link):
-    # Github : https://github.com/(owner)/(repo)
-    # npm : https://www.npmjs.com/package/(package)/v/(version)
-    # npm2 : https://www.npmjs.com/package/@(group)/(package)/v/(version)
-    # pypi : https://pypi.org/project/(oss_name)/(version)
-    # pypi2 : https://files.pythonhosted.org/packages/source/(alphabet)/(oss_name)/(oss_name)-(version).tar.gz
-    # Maven: https://mvnrepository.com/artifact/(group)/(artifact)/(version)
-    # pub: https://pub.dev/packages/(package)/versions/(version)
-    # Cocoapods: https://cocoapods.org/(package)
-    pkg_pattern = {
-        "github": r'https?:\/\/github.com\/([^\/]+)\/([^\/\.]+)(\.git)?',
-        "pypi": r'https?:\/\/pypi\.org\/project\/([^\/]+)[\/]?([^\/]*)',
-        "pypi2": r'https?:\/\/files\.pythonhosted\.org\/packages\/source\/[\w]\/([^\/]+)\/[\S]+-([^\-]+)\.tar\.gz',
-        "maven": r'https?:\/\/mvnrepository\.com\/artifact\/([^\/]+)\/([^\/]+)\/?([^\/]*)',
-        "npm": r'https?:\/\/www\.npmjs\.com\/package\/([^\/\@]+)(?:\/v\/)?([^\/]*)',
-        "npm2": r'https?:\/\/www\.npmjs\.com\/package\/(\@[^\/]+\/[^\/]+)(?:\/v\/)?([^\/]*)',
-        "pub": r'https?:\/\/pub\.dev\/packages\/([^\/]+)(?:\/versions\/)?([^\/]*)',
-        "pods": r'https?:\/\/cocoapods\.org\/pods\/([^\/]+)'
-    }
-    oss_name = ""
-    oss_version = ""
-    if link.startswith("www."):
-        link = link.replace("www.", "https://www.", 1)
-    for key, value in pkg_pattern.items():
-        p = re.compile(value)
-        match = p.match(link)
-        if match:
-            try:
-                origin_name = match.group(1)
-                if key == "github":
-                    repo = match.group(2)
-                    oss_name = f"{origin_name}-{repo}"
-                elif (key == "pypi") or (key == "pypi2"):
-                    oss_name = f"pypi:{origin_name}"
-                    oss_name = re.sub(r"[-_.]+", "-", oss_name).lower()
-                    oss_version = match.group(2)
-                elif key == "maven":
-                    artifact = match.group(2)
-                    oss_name = f"{origin_name}:{artifact}"
-                    origin_name = oss_name
-                    oss_version = match.group(3)
-                elif key == "npm" or key == "npm2":
-                    oss_name = f"npm:{origin_name}"
-                    oss_version = match.group(2)
-                elif key == "pub":
-                    oss_name = f"pub:{origin_name}"
-                    oss_version = match.group(2)
-                elif key == "pods":
-                    oss_name = f"cocoapods:{origin_name}"
-            except Exception as ex:
-                logger.info(f"extract_name_version_from_link {key}:{ex}")
-            if oss_name and (not oss_version):
-                if key in ["pypi", "maven", "npm", "npm2", "pub"]:
-                    oss_version, link = get_latest_package_version(link, key, origin_name)
-                    logger.debug(f'Try to download with the latest version:{link}')
-            break
-    return oss_name, oss_version, link
-
-
-def get_latest_package_version(link, pkg_type, oss_name):
-    find_version = ''
-    link_with_version = link
-
-    try:
-        if pkg_type in ['npm', 'npm2']:
-            stderr, stdout = npm_run('view', oss_name, 'version')
-            if stdout:
-                find_version = stdout.strip()
-            link_with_version = f'https://www.npmjs.com/package/{oss_name}/v/{find_version}'
-        elif pkg_type == 'pypi':
-            find_version = str(latest(oss_name, at='pip', output_format='version', pre_ok=True))
-            link_with_version = f'https://pypi.org/project/{oss_name}/{find_version}'
-        elif pkg_type == 'maven':
-            maven_response = requests.get(f'https://api.deps.dev/v3alpha/systems/maven/packages/{oss_name}')
-            if maven_response.status_code == 200:
-                find_version = maven_response.json().get('versions')[-1].get('versionKey').get('version')
-            oss_name = oss_name.replace(':', '/')
-            link_with_version = f'https://mvnrepository.com/artifact/{oss_name}/{find_version}'
-        elif pkg_type == 'pub':
-            pub_response = requests.get(f'https://pub.dev/api/packages/{oss_name}')
-            if pub_response.status_code == 200:
-                find_version = pub_response.json().get('latest').get('version')
-            link_with_version = f'https://pub.dev/packages/{oss_name}/versions/{find_version}'
-    except Exception as e:
-        logger.debug(f'Fail to get latest package version({link}:{e})')
-    return find_version, link_with_version
-
-
 def overwrite_excel(excel_file_path, oss_name, column_name='OSS Name'):
     if oss_name != "":
         try:

src/fosslight_scanner/fosslight_scanner.py

@@ -23,7 +23,7 @@
 from fosslight_util.output_format import check_output_format
 from fosslight_prechecker._precheck import run_lint as prechecker_lint
 from .common import (copy_file, call_analysis_api,
-                     overwrite_excel, extract_name_version_from_link,
+                     overwrite_excel,
                      merge_yamls, correct_scanner_result,
                      create_scancodejson)
 from fosslight_util.write_excel import merge_excels
@@ -255,14 +255,16 @@ def download_source(link, out_dir):
     start_time = datetime.now().strftime('%Y%m%d_%H%M%S')
     success = False
     temp_src_dir = ""
+    oss_name = ""
+    oss_version = ""
     try:
         success, final_excel_dir, result_log = init(out_dir)
         temp_src_dir = os.path.join(
             _output_dir, SRC_DIR_FROM_LINK_PREFIX + start_time)
 
         link = link.strip()
         logger.info(f"Link to download: {link}")
-        success, msg = cli_download_and_extract(
+        success, msg, oss_name, oss_version = cli_download_and_extract(
             link, temp_src_dir, _output_dir)
 
         if success:
@@ -273,7 +275,7 @@ def download_source(link, out_dir):
     except Exception as ex:
         success = False
         logger.error(f"Failed to analyze from link: {ex}")
-    return success, temp_src_dir
+    return success, temp_src_dir, oss_name, oss_version
 
 
 def init(output_path="", make_outdir=True):
@@ -394,8 +396,7 @@ def run_main(mode, path_arg, dep_arguments, output_file_or_dir, file_format, url
 
                 if url_to_analyze != "":
                     remove_downloaded_source = True
-                    default_oss_name, default_oss_version, url_to_analyze = extract_name_version_from_link(url_to_analyze)
-                    success, src_path = download_source(url_to_analyze, output_path)
+                    success, src_path, default_oss_name, default_oss_version = download_source(url_to_analyze, output_path)
 
                 if output_extension == ".yaml":
                     correct_mode = False