Merge pull request #95 from fosslight/temp_jk

Remove the code of filtering fille name in convert mode

Author: bjk7119

requirements.txt

@@ -3,4 +3,4 @@ binaryornot
 requests
 reuse
 PyYAML
-fosslight_util>=1.4.3
+fosslight_util>=1.4.7

src/fosslight_oss_pkg/_convert.py

@@ -10,10 +10,10 @@
 from datetime import datetime
 from pathlib import Path
 from yaml import safe_dump
-from fosslight_prechecker._help import print_help_msg
 from fosslight_util.constant import LOGGER_NAME
 from fosslight_util.set_log import init_log
 from fosslight_util.output_format import check_output_format
+from fosslight_util.parsing_yaml import find_sbom_yaml_files
 from ._parsing_excel import convert_excel_to_yaml, convert_yml_to_excel
 
 CUSTOMIZED_FORMAT_FOR_PRECHECKER = {'yaml': '.yaml', 'excel': '.xlsx'}
@@ -40,12 +40,19 @@ def find_report_file(path_to_find):
     return ""
 
 
+def check_extension_and_format(file, format):
+    if (file.endswith((".yaml", ".yml")) and format == "yaml") or \
+       (file.endswith(".xlsx") and format == "excel"):
+        logger.error(f"File extension is not matched with input format({format})")
+        sys.exit(1)
+
+
 def convert_report(base_path, output_name, format, need_log_file=True, sheet_names=""):
-    oss_pkg_files = ["oss-pkg-info.yml", "oss-pkg-info.yaml"]
+    oss_yaml_files = []
+    oss_report_files = ""  # TODO: Change to list type for multiple Report files
     file_option_on = False
     convert_yml_mode = False
     convert_excel_mode = False
-    report_to_read = ""
     output_report = ""
     output_yaml = ""
     now = datetime.now().strftime('%Y%m%d_%H-%M-%S')
@@ -68,54 +75,50 @@ def convert_report(base_path, output_name, format, need_log_file=True, sheet_nam
             output_yaml = os.path.join(output_path, output_name)
         else:
             output_report = os.path.join(os.path.abspath(output_path), f"FOSSLight-Report_{now}")
-            output_yaml = os.path.join(os.path.abspath(output_path), f"oss-pkg-info_{now}")
+            output_yaml = os.path.join(os.path.abspath(output_path), f"fosslight-sbom-info_{now}")
     else:
         logger.error(f"Format error - {msg}")
         sys.exit(1)
 
+    if base_path == "":
+        base_path = output_path
+
     if os.path.isdir(base_path):
-        if output_extension == ".yaml":
-            logger.error("Format error - can make only .xlsx file")
-            sys.exit(1)
-        convert_yml_mode = True
+        oss_yaml_files = find_sbom_yaml_files(base_path)
+        if oss_yaml_files:
+            convert_yml_mode = True
     else:
         if base_path != "":
-            if base_path.endswith(".xlsx"):
-                if output_extension == '.xlsx':
-                    logger.error("Format error - can make only .yaml file")
-                    sys.exit(1)
-                p = re.compile(r"[\s\S]*OSS[\s\S]*-Report[\s\S]*.xlsx", re.I)
-                if p.search(base_path):
+            files_to_convert = base_path.split(",")
+            for file in files_to_convert:
+                check_extension_and_format(file, format)
+                if file.endswith(".xlsx"):
                     convert_excel_mode = True
-                    report_to_read = base_path
-            elif base_path.endswith((".yaml", ".yml")):
-                if output_extension == '.yaml':
-                    logger.error("Format error - can make only .xlsx file")
-                    sys.exit(1)
-                p = re.compile(r"oss-pkg-info[\s\S]*.ya?ml", re.I)
-                if p.search(base_path):
-                    oss_pkg_files = base_path.split(',')
+                    oss_report_files = file
+                elif file.endswith((".yaml", ".yml")):
                     convert_yml_mode = True
                     file_option_on = True
-            else:
-                logger.error("Not support file name or extension - only support for FOSSLight-Report*.xlsx or oss-pkg-info*.yaml file")
-                sys.exit(1)
+                    oss_yaml_files.append(file)
+                else:
+                    logger.error("Not support file name or extension")
+                    sys.exit(1)
 
     if not convert_yml_mode and not convert_excel_mode:
         if is_window:
             convert_yml_mode = True
             base_path = os.getcwd()
-            report_to_read = find_report_file(base_path)
-            if report_to_read != "":
+            oss_report_files = find_report_file(base_path)
+            if oss_report_files != "":
                 convert_excel_mode = True
         else:
-            print_help_msg()
+            logger.info("fosslight_prechecker: can't convert anything")
+            logger.info("Try 'fosslight_prechecker -h for more information")
 
     if convert_yml_mode:
-        convert_yml_to_excel(oss_pkg_files, output_report, file_option_on, base_path, is_window)
+        convert_yml_to_excel(oss_yaml_files, output_report, file_option_on, base_path)
 
     if convert_excel_mode:
-        convert_excel_to_yaml(report_to_read, output_yaml, sheet_names)
+        convert_excel_to_yaml(oss_report_files, output_yaml, sheet_names)
 
     try:
         _str_final_result_log = safe_dump(_result_log, allow_unicode=True, sort_keys=True)

src/fosslight_oss_pkg/_parsing_excel.py

@@ -7,7 +7,7 @@
 import sys
 import yaml
 from fosslight_util.constant import LOGGER_NAME
-from fosslight_util.parsing_yaml import find_all_oss_pkg_files, parsing_yml
+from fosslight_util.parsing_yaml import parsing_yml
 from fosslight_util.output_format import write_output_file
 from fosslight_util.write_yaml import create_yaml_with_ossitem
 from fosslight_util.read_excel import read_oss_report
@@ -16,33 +16,30 @@
 IDX_CANNOT_FOUND = -1
 
 
-def convert_yml_to_excel(oss_pkg_files, output_file, file_option_on, base_path, window):
+def convert_yml_to_excel(oss_yaml_files, output_file, file_option_on, base_path):
     items_to_print = []
     sheet_list = {}
 
-    if not file_option_on:
-        oss_pkg_files = find_all_oss_pkg_files(base_path, oss_pkg_files)
-
-    for oss_pkg_file in oss_pkg_files:
+    for yaml_file in oss_yaml_files:
         try:
-            if os.path.isfile(oss_pkg_file):
-                logger.warning(f"Read data from : {oss_pkg_file}")
+            if os.path.isfile(yaml_file):
+                logger.info(f"Read data from : {yaml_file}")
 
                 if file_option_on:
-                    base_path = os.path.dirname(oss_pkg_file)
-                oss_items, _ = parsing_yml(oss_pkg_file, base_path)
+                    base_path = os.path.dirname(yaml_file)
+                oss_items, _ = parsing_yml(yaml_file, base_path)
                 for item in oss_items:
                     items_to_print.extend(item.get_print_array())
         except Exception as ex:
             logger.error(f"Read yaml file: {ex}")
 
     try:
-        sheet_list["SRC_FL_Reuse"] = items_to_print
+        sheet_list["SRC_FL_Prechecker"] = items_to_print
         success, msg, result_file = write_output_file(output_file, '.xlsx',
                                                       sheet_list)
         if success:
             if result_file:
-                logger.info(f"Output: {result_file}")
+                logger.warning(f"Output: {result_file}")
             else:
                 logger.warning("Nothing is detected to convert so output file is not generated.")
         else:

src/fosslight_prechecker/_add.py

@@ -12,7 +12,7 @@
 from yaml import safe_dump
 from fosslight_util.set_log import init_log
 from fosslight_util.spdx_licenses import get_spdx_licenses_json
-from fosslight_util.parsing_yaml import find_all_oss_pkg_files, parsing_yml
+from fosslight_util.parsing_yaml import find_sbom_yaml_files, parsing_yml
 from fosslight_util.output_format import check_output_format
 from datetime import datetime
 from fosslight_prechecker._precheck import precheck_for_project, precheck_for_files, dump_error_msg, get_path_to_find
@@ -298,24 +298,24 @@ def is_exclude_dir(dir_path):
 
 
 def download_oss_info_license(base_path, input_license=""):
-    oss_pkg_files = ["oss-pkg-info.yml", "oss-pkg-info.yaml"]
     license_list = []
     converted_lic_list = []
+    oss_yaml_files = []
     main_parser = reuse_arg_parser()
     prj = Project(base_path)
 
-    found_oss_pkg_files = find_all_oss_pkg_files(base_path, oss_pkg_files)
+    oss_yaml_files = find_sbom_yaml_files(base_path)
 
     if input_license != "":
         license_list.append(input_license)
 
-    if found_oss_pkg_files is None or len(found_oss_pkg_files) == 0:
+    if oss_yaml_files is None or len(oss_yaml_files) == 0:
         logger.info("\n # There is no OSS package Info file in this path\n")
         return
     else:
-        logger.info(f"\n # There is OSS Package Info file(s) : {found_oss_pkg_files}\n")
+        logger.info(f"\n # There is OSS Package Info file(s) : {oss_yaml_files}\n")
 
-    for oss_pkg_file in found_oss_pkg_files:
+    for oss_pkg_file in oss_yaml_files:
         _, license_list = parsing_yml(oss_pkg_file, base_path)
 
     for lic in license_list:

src/fosslight_prechecker/_constant.py

@@ -5,9 +5,9 @@
 
 DEFAULT_EXCLUDE_EXTENSION = ["jar", "png", "exe", "so", "a", "dll", "jpeg", "jpg", "ttf", "lib", "ttc", "pfb",
                              "pfm", "otf", "afm", "dfont", "json"]
-OSS_PKG_INFO_FILES = [r"oss-pkg-info[\s\S]*.yaml", "oss-pkg-info.yml", r"oss-package[\s\S]*.info", "requirement.txt",
-                      "requirements.txt", "package.json", "pom.xml", "build.gradle", "podfile.lock", "cartfile.resolved",
-                      "pubspec.yaml", "package.resolved", "go.mod", r"fosslight-sbom-info[\s\S]*.yaml"]
+OSS_PKG_INFO_FILES = [r"oss-pkg-info[\s\S]*.ya?ml", r"oss-package[\s\S]*.info", r"sbom-info[\s\S]*.ya?ml",
+                      "requirement.txt", "requirements.txt", "package.json", "pom.xml", "build.gradle",
+                      "podfile.lock", "cartfile.resolved", "pubspec.yaml", "package.resolved", "go.mod"]
 HTML_RESULT_EXPAND_LIMIT = 10
 HTML_RESULT_PRINT_LIMIT = 100
 

src/fosslight_prechecker/_result.py

@@ -13,7 +13,7 @@
 from pathlib import Path
 from reuse.project import Project
 from fosslight_prechecker._result_html import result_for_html
-from fosslight_util.parsing_yaml import find_all_oss_pkg_files, parsing_yml
+from fosslight_util.parsing_yaml import find_sbom_yaml_files, parsing_yml
 from fosslight_util.output_format import check_output_format
 import re
 
@@ -286,8 +286,8 @@ def result_for_summary(path_to_find, oss_pkg_info_files, license_missing_files,
             detected_lic.append(lic)
 
     if oss_pkg_info_files:
-        pkg_info_yaml_files = find_all_oss_pkg_files(path_to_find, oss_pkg_info_files)
-        yaml_file = get_only_pkg_info_yaml_file(pkg_info_yaml_files)
+        oss_yaml_files = find_sbom_yaml_files(path_to_find)
+        yaml_file = get_only_pkg_info_yaml_file(oss_yaml_files)
         # Exclude files in yaml
         license_missing_files, copyright_missing_files = exclude_file_in_yaml(path_to_find, yaml_file,
                                                                               set(license_missing_files) - set(oss_pkg_info_files),

tests/convert/oss-pkg-info.yaml

@@ -1,34 +1,36 @@
-libidn:
-- version: "1.5"
-  source name or path:
+Open Source Software Package:
+- name: libidn
+  version: 1.5
+  source: http://ftp.gnu.org/gnu/libidn
+  homepage: https://www.gnu.org/software/libidn
+  license:
+  - GPL-3.0
+  - LGPL-2.1
+  file: 
   - a.c
   - b.c
-  license:
-  - "GPL-3.0"
-  - "LGPL-2.1"
-  download location: "http://ftp.gnu.org/gnu/libidn"
-  homepage: "https://www.gnu.org/software/libidn"
-  copyright text: "Copyright (c) 2002-2007, Simon Josefsson"
-node-backoff:
-- version: "2.5.0"
-  source name or path: "src/*"
-  license: "MIT"
-  download location: "https://github.com/MathieuTurcotte/node-backoff"
-  homepage: "https://www.npmjs.com/package/backoff"
-  copyright text: "Copyright (c) 2012 Mathieu Turcotte"
+  copyright: Copyright (c) 2002-2007, Simon Josefsson
+- name: node-backoff
+  version: 2.5.0
+  source: https://github.com/MathieuTurcotte/node-backoff
+  homepage: https://www.npmjs.com/package/backoff
+  license: MIT
+  file: src/*
+  copyright: Copyright (c) 2012 Mathieu Turcotte
   exclude: True
-rsync:
-- version: "2.6.9"
-  source name or path: "test/tool"
-  license: "GPL-2.0"
-  download location: "https://download.samba.org/pub/rsync/src"
-  homepage: "http://rsync.samba.org"
-- version: "3.1.2"
-  source name or path: "test/tool"
-  license: "GPL-3.0"
-  download location: "https://download.samba.org/pub/rsync/src"
-  homepage: "http://rsync.samba.org"
-  copyright text:
-  - "Copyright (c) 1996 Andrew Tridgell"
-  - "Copyright (c) 1996 Paul Mackerras"
-  - "Copyright (c) 2003-2015 Wayne Davison"
+- name: rsync
+  version: 2.6.9
+  source: https://download.samba.org/pub/rsync/src
+  homepage: http://rsync.samba.org
+  license: GPL-2.0
+  file: test/tool
+- name: rsync
+  version: 3.1.2
+  source: https://download.samba.org/pub/rsync/src
+  homepage: http://rsync.samba.org
+  license: GPL-3.0
+  file: test/tool
+  copyright text: |
+    Copyright (c) 1996 Andrew Tridgell
+    Copyright (c) 1996 Paul Mackerras
+    Copyright (c) 2003-2015 Wayne Davison

tests/convert/oss-pkg-info_old.yaml

@@ -0,0 +1,34 @@
+libidn:
+- version: "1.5"
+  source name or path:
+  - a.c
+  - b.c
+  license:
+  - "GPL-3.0"
+  - "LGPL-2.1"
+  download location: "http://ftp.gnu.org/gnu/libidn"
+  homepage: "https://www.gnu.org/software/libidn"
+  copyright text: "Copyright (c) 2002-2007, Simon Josefsson"
+node-backoff:
+- version: "2.5.0"
+  source name or path: "src/*"
+  license: "MIT"
+  download location: "https://github.com/MathieuTurcotte/node-backoff"
+  homepage: "https://www.npmjs.com/package/backoff"
+  copyright text: "Copyright (c) 2012 Mathieu Turcotte"
+  exclude: True
+rsync:
+- version: "2.6.9"
+  source name or path: "test/tool"
+  license: "GPL-2.0"
+  download location: "https://download.samba.org/pub/rsync/src"
+  homepage: "http://rsync.samba.org"
+- version: "3.1.2"
+  source name or path: "test/tool"
+  license: "GPL-3.0"
+  download location: "https://download.samba.org/pub/rsync/src"
+  homepage: "http://rsync.samba.org"
+  copyright text:
+  - "Copyright (c) 1996 Andrew Tridgell"
+  - "Copyright (c) 1996 Paul Mackerras"
+  - "Copyright (c) 2003-2015 Wayne Davison"