Merge pull request #36 from fosslight/develop

Add function to get spdx licenses

Author: dd-jy

README.md

@@ -10,8 +10,8 @@ It is a package that supports common utils used by FOSSLight Scanner.
 
 ## Features 
 1. It simplifies the logger setup.
-2. It easily outputs csv file and excel file in FOSSLight Report format.
-3. It provides a simple function to create a text file.
+2. It provides a simple function to create a output file.
+3. It provides a spdx license list with json format.
 4. It defines common constant variables.
 5. It provides a thread that prints the spinner.
 6. Download source code.
@@ -65,31 +65,29 @@ def test():
 ```
 
 
-### 2. Write csv and excel files (tests/test_excel.py)
+### 2. Write result files (tests/test_output_format.py)
 ```
-from fosslight_util._write_excel import write_excel_and_csv
-
+from fosslight_util.output_format import write_output_file
 
+# 2nd param : output file format
+#            => file format(excel: .xlsx, csv: .csv, opossum: .json)
 def test():
     sheet_contents = {'SRC':[['run_scancode.py', 'fosslight_source',
                         '3.0.6', 'Apache-2.0',  'https://github.com/LGE-OSS/fosslight_source', 'https://github.com/LGE-OSS/fosslight_source', 'Copyright (c) 2021 LG Electronics, Inc.', 'Exclude', 'Comment message'],
                        ['dependency_unified.py', 'fosslight_dependency',
                         '3.0.6', 'Apache-2.0',  'https://github.com/LGE-OSS/fosslight_dependency', 'https://github.com/LGE-OSS/fosslight_dependency', 'Copyright (c) 2020 LG Electronics, Inc.', '', '']],
                       'BIN':[['askalono.exe', 'askalono',
                         '0.4.3', 'Apache-2.0', 'https://github.com/jpeddicord/askalono', '', 'Copyright (c) 2018 Amazon.com, Inc. or its affiliates.', '', '']]}
-
-    success, msg = write_excel_and_csv(
-        'test_result/excel/FOSSLight-Report', sheet_contents)
+    success, msg = write_output_file('test_result/excel/FOSSLight-Report', '.xlsx', sheet_contents)
 ```
 
-### 3. Write a text file (tests/test_text.py)
+### 3. Get spdx licenses (tests/test_spdx_licenses.py)
 ```
-from fosslight_util.write_txt import write_txt_file
+from fosslight_util.spdx_licenses import get_spdx_licenses_json
 
 
 def test():
-    success, error_msg = write_txt_file("test_result/txt/test.txt",
-                                       "Testing - Writing text in a file.")
+    success, error_msg, licenses = get_spdx_licenses_json()
 ```
 
 ### 4. Load common constant (tests/_print_log_with_another_logger.py)

setup.py

@@ -31,7 +31,7 @@
                      "Programming Language :: Python :: 3.8",
                      "Programming Language :: Python :: 3.9", ],
         install_requires=required,
-        package_data={'fosslight_util': ['resources/frequentLicenselist.json']},
+        package_data={'fosslight_util': ['resources/frequentLicenselist.json', 'resources/licenses.json']},
         include_package_data=True,
         entry_points={
             "console_scripts": [

src/fosslight_util/_create_frequenctLicenses.py renamed to src/fosslight_util/spdx_licenses.py

@@ -14,26 +14,42 @@
 _frequentLicenselist_file = 'frequentLicenselist.json'
 
 
-def create_frequentlicenses():
+def get_spdx_licenses_json():
     success = True
     error_msg = ''
     licenses = ''
+    # licenses : https://github.com/spdx/license-list-data/blob/v3.14/json/licenses.json
     licenses_file = os.path.join(_resources_dir, _licenses_json_file)
-    frequentLicenses = {}
-    _frequentLicenses_key = 'frequentLicenses'
 
-    # spdx_txt_url = _spdx_txt_base_url + version + /text/ + licenseId + '.txt'
-    _spdx_txt_base_url = 'https://raw.githubusercontent.com/spdx/license-list-data/'
     try:
         base_dir = sys._MEIPASS
     except Exception:
         base_dir = os.path.dirname(__file__)
-    # https://github.com/spdx/license-list-data/blob/v3.14/json/licenses.json
-    file_withpath = os.path.join(base_dir, licenses_file)
 
+    file_withpath = os.path.join(base_dir, licenses_file)
     try:
         with open(file_withpath, 'r') as f:
             licenses = json.load(f)
+    except Exception as e:
+        success = False
+        error_msg = 'Failed to open ' + file_withpath + ': ' + str(e)
+
+    return success, error_msg, licenses
+
+
+def create_frequentlicenses():
+    success = True
+    error_msg = ''
+    licenses = ''
+    frequentLicenses = {}
+    _frequentLicenses_key = 'frequentLicenses'
+
+    # spdx_txt_url = _spdx_txt_base_url + version + /text/ + licenseId + '.txt'
+    _spdx_txt_base_url = 'https://raw.githubusercontent.com/spdx/license-list-data/'
+
+    try:
+        success, error_msg, licenses = get_spdx_licenses_json()
+        if success:
             version = licenses['licenseListVersion']
             frequentLicenses[_frequentLicenses_key] = []
             for lic in licenses['licenses']:
@@ -55,7 +71,7 @@ def create_frequentlicenses():
 
     except Exception as e:
         success = False
-        error_msg = 'Failed to open and parse ' + licenses_file
+        error_msg = 'Failed to open and parse licenses.json'
         print(error_msg, ": ", e)
         print(traceback.format_exc())
 

tests/test_output_format.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2021 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+from fosslight_util.output_format import write_output_file
+from fosslight_util.set_log import init_log
+
+
+def main():
+    logger, _result_log = init_log("test_result/output_format/log_write_output.txt")
+
+    sheet_list = {'SRC_FL_Source': [
+                  ['test/lib/babel-polyfill.js', '', '', 'bsd-3-clause,facebook-patent-rights-2', '', '',
+                   'Copyright (c) 2014, Facebook, Inc.', 'Exclude', ''],
+                  ['requirements.txt', '', '', 'MIT', 'https://pypi.org/project/future/0.18.2', '', '', '', ''],
+                  ['bower.json', '', '', 'mit', '', '', '', '', ''],
+                  ['LICENSE', '', '', 'mit', '', '', 'Copyright (c) 2016-2021, The Cytoscape Consortium', '', ''],
+                  ['license-update.js', '', '', 'mit', '', '', 'Copyright (c) 2016-$ year, The Cytoscape Consortium', '', ''],
+                  ['package.json', '', '', 'mit', '', '', '', '', ''], ['README.md', '', '', 'mit', '', '', '', '', ''],
+                  ['dist/cytoscape.cjs.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,Copyright (c) 2016-2021,c \
+                   The Cytoscape Consortium,copyright Koen Bok,Copyright (c) 2013-2014 Ralf S. Engelschall \
+                   (http://engelschall.com)', '', ''],
+                  ['dist/cytoscape.esm.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,Copyright (c) 2016-2021,\
+                   The Cytoscape Consortium,copyright Koen Bok,Copyright (c) 2013-2014 Ralf S. Engelschall \
+                   (http://engelschall.com)', '', ''],
+                  ['dist/cytoscape.esm.min.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,copyright Koen Bok, \
+                   Copyright (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''],
+                  ['dist/cytoscape.min.js', '', '', 'mit',
+                   '', '', 'Copyright Gaetan Renaudeau,Copyright (c) 2016-2021, The Cytoscape Consortium,copyright Koen Bok,Copyright \
+                  (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''],
+                  ['dist/cytoscape.umd.js', '', '', 'mit', '', '',
+                   'Copyright Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors,Copyright jQuery Foundation \
+                   and other contributors <https://jquery.org/>,Copyright (c) 2016-2021, The Cytoscape Consortium,copyright Koen\
+                   Bok,Copyright Gaetan Renaudeau,Copyright (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''],
+                  ['documentation/css/highlight/monokai_sublime.css', '', '', 'mit', '', '', '', '', ''],
+                  ['documentation/js/cytoscape.min.js', '', '', 'mit', '', '', 'Copyright Gaetan Renaudeau,\
+                   Copyright (c) 2016-2021, The Cytoscape Consortium,copyright Koen Bok, \
+                   Copyright (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)', '', ''],
+                  ['documentation/md/links.md', '', '', 'mit', '', '', '', '', ''],
+                  ['src/event.js', '', '', 'mit', '', '', '', '', '']],
+                  'BIN_FL_Binary': [
+                  ['askalono_macos', 'askalono', '', 'Apache-2.0', '', '', '', '', ''],
+                  ['test/askalono_macos', 'askalono', '', 'Apache-2.0', '', '', '', 'Exclude', '']],
+                  'SRC_FL_Dependency': [
+                  ['requirements.txt', 'pypi:future', '0.18.2', 'MIT', 'https://pypi.org/project/future/0.18.2',
+                   'https://python-future.org', '', '', ''],
+                  ['requirements.txt', 'pypi:numpy', '1.19.5', 'BSD-3-Clause-Open-MPI,GCC-exception-3.1,GPL-3.0',
+                   'https://pypi.org/project/numpy/1.19.5', 'https://www.numpy.org', '', '', ''],
+                  ['requirements.txt', 'pypi:pandas', '1.1.5', 'BSD-3-Clause', 'https://pypi.org/project/pandas/1.1.5',
+                   'https://pandas.pydata.org', '', '', '']]}
+
+    logger.warning("TESTING - Writing an excel output")
+    success, msg = write_output_file(
+        'test_result/output_format/FL-TEST_Excel.xlsx', '.xlsx', sheet_list)
+    logger.warning("Result:" + str(success) + ", error_msg:" + msg)
+
+    logger.warning("TESTING - Writing an opossum output")
+    success, msg = write_output_file(
+        'test_result/output_format/FL-TEST_opossum.json', '.json', sheet_list)
+    logger.warning("Result:" + str(success) + ", error_msg:" + msg)
+
+
+if __name__ == '__main__':
+    main()

tests/test_spdx_licenses.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2021 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+from fosslight_util.set_log import init_log
+from fosslight_util.spdx_licenses import get_spdx_licenses_json
+
+
+def main():
+    logger, _result_log = init_log("test_result/spdx_licenses/log_spdx_licenses.txt")
+    logger.warning("TESTING - Get spdx licenses")
+
+    success, error_msg, licenses = get_spdx_licenses_json()
+    logger.warning("Result:" + str(success) + ", error_msg:" + error_msg)
+
+
+if __name__ == '__main__':
+    main()

tox.ini

@@ -47,6 +47,10 @@ commands =
   fosslight_download -s "https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-databind/2.12.2" -t test_wget -d test_logs
   fosslight_download -s "https://pub.dev/packages/file/versions/5.2.1" -t test_wget -d test_logs
   ls test_wget/
+  # Test - write output file
+  python tests/test_output_format.py
+  # Test - get spdx licenses
+  python tests/test_spdx_licenses.py
 
 [testenv:release]
 deps =
@@ -80,5 +84,9 @@ commands =
   # Test - downloading source
   python tests/test_download.py
   ls test_result/download
+  # Test - write output file
+  python tests/test_output_format.py
+  # Test - get spdx licenses
+  python tests/test_spdx_licenses.py
   # Test - check PEP8
   pytest -v --flake8