Extract the OSS name from the link

Author: soimkim

src/fosslight_scanner/common.py

@@ -7,6 +7,8 @@
 import sys
 import logging
 from shutil import copy
+import re
+import pandas as pd
 import fosslight_util.constant as constant
 
 logger = logging.getLogger(constant.LOGGER_NAME)
@@ -68,3 +70,75 @@ def call_analysis_api(path_to_run, str_run_start, return_idx, func, *args):
     if not result:
         result = []
     return success, result
+
+
+def extract_name_from_link(link):
+    # Github : https://github.com/(owner)/(repo)
+    # npm : www.npmjs.com/package/(package)
+    # npm : https://www.npmjs.com/package/@(group)/(package)
+    # pypi : https://pypi.org/project/(oss_name)
+    # Maven: https://mvnrepository.com/artifact/(group)/(artifact)
+    # pub: https://pub.dev/packages/(package)
+    # Cocoapods: https://cocoapods.org/(package)
+    pkg_pattern = {
+        "github": r'https?:\/\/github.com\/([^\/]+)\/([^\/\.]+)(\.git)?',
+        "pypi": r'https?:\/\/pypi\.org\/project\/([^\/]+)',
+        "maven": r'https?:\/\/mvnrepository\.com\/artifact\/([^\/]+)\/([^\/]+)',
+        "npm": r'https?:\/\/www\.npmjs\.com\/package\/([^\/]+)(\/[^\/]+)?',
+        "pub": r'https?:\/\/pub\.dev\/packages\/([^\/]+)',
+        "pods": r'https?:\/\/cocoapods\.org\/pods\/([^\/]+)'
+    }
+    oss_name = ""
+    if link.startswith("www."):
+        link = link.replace("www.", "https://www.", 1)
+    for key, value in pkg_pattern.items():
+        try:
+            p = re.compile(value)
+            match = p.match(link)
+            if match:
+                group = match.group(1)
+                if key == "github":
+                    repo = match.group(2)
+                    oss_name = f"{group}-{repo}"
+                    break
+                elif key == "pypi":
+                    oss_name = f"pypi:{group}"
+                    break
+                elif key == "maven":
+                    artifact = match.group(2)
+                    oss_name = f"{group}:{artifact}"
+                    break
+                elif key == "npm":
+                    if group.startswith("@"):
+                        pkg = match.group(2)
+                        oss_name = f"npm:{group}{pkg}"
+                    else:
+                        oss_name = f"npm:{group}"
+                    break
+                elif key == "pub":
+                    oss_name = f"pub:{group}"
+                    break
+                elif key == "pods":
+                    oss_name = f"cocoapods:{group}"
+                    break
+        except Exception as ex:
+            logger.debug(f"extract_name_from_link_{key}:{ex}")
+    return oss_name
+
+
+def overwrite_excel(excel_file_path, oss_name, column_name='OSS Name'):
+    if oss_name != "":
+        try:
+            files = os.listdir(excel_file_path)
+            for file in files:
+                if file.endswith(".xlsx"):
+                    file = os.path.join(excel_file_path, file)
+                    excel_file = pd.ExcelFile(file, engine='openpyxl')
+
+                    for sheet_name in excel_file.sheet_names:
+                        df = pd.read_excel(file, sheet_name=sheet_name, engine='openpyxl')
+                        updated = (df[column_name] == '') | (df[column_name].isnull())
+                        df.loc[updated, column_name] = oss_name
+                        df.to_excel(file, sheet_name=sheet_name, index=False)
+        except Exception as ex:
+            logger.debug(f"overwrite_excel:{ex}")

src/fosslight_scanner/fosslight_scanner.py

@@ -23,7 +23,8 @@
 import fosslight_util.constant as constant
 from fosslight_util.output_format import check_output_format
 from fosslight_reuse._fosslight_reuse import run_lint as reuse_lint
-from .common import copy_file, call_analysis_api
+from .common import (copy_file, call_analysis_api,
+                     overwrite_excel, extract_name_from_link)
 from fosslight_util.write_excel import merge_excels
 
 OUTPUT_EXCEL_PREFIX = "FOSSLight-Report_"
@@ -90,7 +91,7 @@ def run_dependency(path_to_analyze, output_file_with_path, params=""):
 def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
                 run_src=True, run_bin=True, run_dep=True, run_reuse=True,
                 remove_src_data=True, result_log={}, output_file="",
-                output_extension="", num_cores=-1, db_url=""):
+                output_extension="", num_cores=-1, db_url="", default_oss_name=""):
     create_csv = False
     final_excel_dir = output_path
     success = True
@@ -157,6 +158,8 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
     try:
         output_file_without_ext = os.path.join(final_excel_dir, output_file)
         final_report = f"{output_file_without_ext}{output_extension}"
+        if remove_src_data:
+            overwrite_excel(_output_dir, default_oss_name)
         success, output_files = merge_excels(_output_dir, final_report, create_csv)
 
         if success and output_files:
@@ -187,7 +190,7 @@ def download_source(link, out_dir):
     try:
         success, final_excel_dir, result_log = init(out_dir)
         temp_src_dir = os.path.join(
-            _output_dir, SRC_DIR_FROM_LINK_PREFIX+start_time)
+            _output_dir, SRC_DIR_FROM_LINK_PREFIX + start_time)
 
         logger.info(f"Link to download: {link}")
         success, msg = cli_download_and_extract(
@@ -254,6 +257,7 @@ def run_main(mode, src_path, dep_arguments, output_file_or_dir, file_format, url
 
             if url_to_analyze != "":
                 remove_downloaded_source = True
+                default_oss_name = extract_name_from_link(url_to_analyze)
                 success, src_path = download_source(url_to_analyze, output_path)
 
             if mode == "reuse":
@@ -274,7 +278,7 @@ def run_main(mode, src_path, dep_arguments, output_file_or_dir, file_format, url
                 run_scanner(src_path, dep_arguments, output_path, keep_raw_data,
                             run_src, run_bin, run_dep, run_reuse,
                             remove_downloaded_source, {}, output_file,
-                            output_extension, num_cores, db_url)
+                            output_extension, num_cores, db_url, default_oss_name)
 
     except Exception as ex:
         logger.warning(str(ex))

tox.ini

@@ -6,7 +6,6 @@ skipdist = true
 
 [testenv]
 install_command = pip install {opts} {packages}
-basepython= python3.6
 whitelist_externals = cat
                       cp
                       rm