Fixed #485 - Implement gen_license command

Signed-off-by: Chin Yeung Li <[email protected]>

Author: chinyeungli

CHANGELOG.rst

@@ -16,6 +16,7 @@
     * Support Excel file format for `inventory`, `gen` and `attrib`
     * Add 'spdx_license_key' support
     * Add option to save error log in `check` command
+    * New `gen_license` option
 
 2021-04-02
     Release 6.0.0

docs/source/general.rst

@@ -17,6 +17,8 @@ AboutCode Toolkit is a tool for your software development team to document your
 
 -   **gen**: Create ABOUT file(s) from a Software Inventory file (.csv, .json or .xlsx format) which is typically created from a software audit, and insert these AboutCode Toolkit files into your codebase. You can regenerate the AboutCode Toolkit files from a new Software Inventory file whenever you make changes.
 
+-   **gen_license**: Fetch licenses in the license_expression field and save to the output location.
+
 -   **inventory**: Generate a Software Inventory list (.csv, .json or .xlsx format) from your codebase based on ABOUT file(s). Note that this Software Inventory will only include components that have AboutCode Toolkit data. In another word, if you do not create AboutCode Toolkit files for your own original software components, these components will not show up in the generated inventory.
 
 -   **transform**: A command to transform an input CSV/JSON/Excel by applying renaming and/or filtering and then output to a new CSV/JSON/Excel file.

docs/source/reference.rst

@@ -185,6 +185,19 @@ Details
 
         ..  code-block:: none
 
+                ---djc
+
+                    Validate license_expression from a DejaCode License.
+
+                    This option requires 2 parameters:
+                        api_url - URL to the DJE License Library.
+                        api_key - Hash key to authenticate yourself in the API.
+
+                    In addition, the input needs to have the 'license_expression' field.
+                    (Please contact nexB to get the api_* value for this feature)
+
+                $ about check --djc 'api_url' 'api_key' /home/project/about_files/
+
                 --log
 
                     This option save the error log to the defined location
@@ -379,6 +392,60 @@ Details
                     This option tells the tool to show all errors found.
                     The default behavior will only show 'CRITICAL', 'ERROR', and 'WARNING'
 
+gen_license
+===========
+
+Syntax
+------
+
+        ..  code-block:: none
+
+                about gen_license [OPTIONS] LOCATION OUTPUT
+
+                LOCATION: Path to a JSON/CSV/Excel/.ABOUT file(s)
+                OUTPUT: Path to a directory where license files are saved.
+
+Options
+-------
+
+        ..  code-block:: none
+
+                --djc api_url api_key   Fetch licenses data from DejaCode License
+                                        Library and create <license>.LICENSE to the
+                                        OUTPUT location.
+
+                 --verbose              Show all the errors and warning.
+                -h, --help              Show this message and exit.
+
+Purpose
+-------
+
+Fetch licenses in the license_expression field and save to the output location.
+
+Details
+^^^^^^^
+
+        ..  code-block:: none
+
+                --djc
+
+                    Fetch licenses text from a DejaCode API, and create <license>.LICENSE to the 
+                    OUTPUT Location using the data fetched from the DejaCode License Library.
+
+                    This option requires 2 parameters:
+                        api_url - URL to the DJE License Library.
+                        api_key - Hash key to authenticate yourself in the API.
+
+                    In addition, the input needs to have the 'license_expression' field.
+                    (Please contact nexB to get the api_* value for this feature)
+
+                $ about gen_license --djc 'api_url' 'api_key' LOCATION OUTPUT
+
+                --verbose
+
+                    This option tells the tool to show all errors found.
+                    The default behavior will only show 'CRITICAL', 'ERROR', and 'WARNING'
+
 inventory
 =========
 

src/attributecode/cmd.py

@@ -48,6 +48,7 @@
 from attributecode.util import filter_errors
 from attributecode.util import get_temp_dir
 from attributecode.util import get_file_text
+from attributecode.util import write_licenses
 
 __copyright__ = """
     Copyright (c) nexB Inc and others. All rights reserved.
@@ -276,6 +277,85 @@ def gen(location, output, android, fetch_license, fetch_license_djc, reference,
         click.echo(msg)
     sys.exit(errors_count)
 
+
+######################################################################
+# gen_license subcommand
+######################################################################
+
+@about.command(cls=AboutCommand,
+    short_help='Fetch and save all the licenses in the license_expression field to a directory.')
+
+@click.argument('location',
+    required=True,
+    metavar='LOCATION',
+    type=click.Path(
+        exists=True, file_okay=True, dir_okay=True, readable=True, resolve_path=True))
+
+@click.argument('output',
+    required=True,
+    metavar='OUTPUT',
+    type=click.Path(exists=True, file_okay=False, writable=True, resolve_path=True))
+
+@click.option('--djc',
+    nargs=2,
+    type=str,
+    metavar='api_url api_key',
+    help='Fetch licenses from a DejaCode License Library.')
+
+@click.option('--verbose',
+    is_flag=True,
+    help='Show all error and warning messages.')
+
+@click.help_option('-h', '--help')
+def gen_license(location, output, djc, verbose):
+    """
+Fetch licenses in the license_expression field and save to the output location.
+
+LOCATION: Path to a JSON/CSV/Excel/.ABOUT file(s)
+
+OUTPUT: Path to a directory where license files are saved.
+    """
+    print_version()
+
+    if location.endswith('.csv') or location.endswith('.json') or location.endswith('.xlsx'):
+        _errors, abouts = load_inventory(
+            location=location
+        )
+    else:
+        _errors, abouts = collect_inventory(location)
+
+
+    log_file_loc = os.path.join(output, 'error.log')
+    api_url = ''
+    api_key = ''
+    errors = []
+    if djc:
+        # Strip the ' and " for api_url, and api_key from input
+        api_url = djc[0].strip("'").strip('"')
+        api_key = djc[1].strip("'").strip('"')
+
+    click.echo('Fetching licenses...')
+    license_dict, lic_errors = pre_process_and_fetch_license_dict(abouts, api_url, api_key)
+    if lic_errors:
+        errors.extend(lic_errors)
+
+    # A dictionary with license file name as the key and context as the value
+    lic_dict_output = {}
+    for key in license_dict:
+        if not key in lic_dict_output:
+            lic_filename = license_dict[key][1]
+            lic_context = license_dict[key][2]
+            lic_dict_output[lic_filename] = lic_context 
+
+    write_errors = write_licenses(lic_dict_output, output)
+    if write_errors:
+        errors.extend(write_errors)
+
+    errors = unique(errors)
+    severe_errors_count = report_errors(errors, quiet=False, verbose=verbose, log_file_loc=log_file_loc)
+    sys.exit(severe_errors_count)
+
+
 ######################################################################
 # attrib subcommand
 ######################################################################
@@ -741,6 +821,7 @@ def report_errors(errors, quiet, verbose, log_file_loc=None):
         log_msgs, _ = get_error_messages(errors, quiet=False, verbose=True)
         with io.open(log_file_loc, 'w', encoding='utf-8', errors='replace') as lf:
             lf.write('\n'.join(log_msgs))
+        click.echo("Error log: " + log_file_loc)
     return severe_errors_count
 
 

src/attributecode/util.py

@@ -707,6 +707,25 @@ def load_excel(location):
         results.append(row_dict)
     return errors, results
 
+def write_licenses(lic_dict, location):
+    import io
+
+    loc = to_posix(location)
+    errors = []
+
+    if not posixpath.exists(loc):
+        os.makedirs(add_unc(loc))
+    try:
+        for lic in lic_dict:
+            output_location = posixpath.join(loc, lic)
+            with io.open(output_location, 'w', encoding='utf-8', errors='replace') as out:
+                out.write(lic_dict[lic])
+    except Exception as e:
+        msg = str(e)
+        errors.append(Error(CRITICAL, msg))
+    return errors
+
+
 """
 Return True if a string s  name is safe to use as an attribute name.
 """

tests/test_cmd.py

@@ -367,6 +367,11 @@ def test_about_gen_help_text():
         'test_cmd/help/about_gen_help.txt', regen=False)
 
 
+def test_about_gen_license_help_text():
+    check_about_stdout(
+        ['gen-license', '--help'],
+        'test_cmd/help/about_gen_license_help.txt', regen=False)
+
 def test_about_check_help_text():
     check_about_stdout(
         ['check', '--help'],

tests/testdata/test_cmd/help/about_gen_license_help.txt

@@ -0,0 +1,13 @@
+Usage: about gen-license [OPTIONS] LOCATION OUTPUT
+
+  Fetch licenses in the license_expression field and save to the output
+  location.
+
+  LOCATION: Path to a JSON/CSV/Excel/.ABOUT file(s)
+
+  OUTPUT: Path to a directory where license files are saved.
+
+Options:
+  --djc api_url api_key  Fetch licenses from a DejaCode License Library.
+  --verbose              Show all error and warning messages.
+  -h, --help             Show this message and exit.

tests/testdata/test_cmd/help/about_help.txt

@@ -19,6 +19,8 @@ Commands:
                       report errors and warnings.
   collect-redist-src  Collect redistributable sources.
   gen                 Generate .ABOUT files from an inventory as CSV/JSON/Excel.
+  gen-license         Fetch and save all the licenses in the license_expression
+                      field to a directory.
   inventory           Collect the inventory of .ABOUT files to a CSV/JSON/Excel
                       file.
   transform           Transform a CSV/JSON/Excel by applying renamings, filters