Fixed #488 - Improve performance for gen_license

 * Created new functions to create ABOUT objects without doing any
validation such as filename, duplicated keys/rows, file fields etc.. as
the only needed information for this `gen_license` is the
`license_expression`.

Signed-off-by: Chin Yeung Li <[email protected]>

Author: chinyeungli

docs/source/reference.rst

@@ -413,7 +413,8 @@ Options
                 --djc api_url api_key   Fetch licenses data from DejaCode License
                                         Library and create <license>.LICENSE to the
                                         OUTPUT location.
-
+                --scancode              Indicate the input JSON file is from
+                                        scancode_toolkit.
                  --verbose              Show all the errors and warning.
                 -h, --help              Show this message and exit.
 
@@ -441,6 +442,12 @@ Details
 
                 $ about gen_license --djc 'api_url' 'api_key' LOCATION OUTPUT
 
+                --scancode
+
+                    Indicates the JSON input is from scancode toolkit license detection
+
+                $ about gen_license --scancode /home/project/scancode-license-detection.json OUTPUT
+
                 --verbose
 
                     This option tells the tool to show all errors found.

src/attributecode/cmd.py

@@ -36,8 +36,9 @@
 from attributecode.attrib import DEFAULT_TEMPLATE_FILE, DEFAULT_LICENSE_SCORE
 from attributecode.attrib import generate_and_save as generate_attribution_doc
 from attributecode.gen import generate as generate_about_files, load_inventory
-from attributecode.model import collect_inventory, get_copy_list
+from attributecode.model import collect_inventory, collect_abouts_license_expression, collect_inventory_license_expression
 from attributecode.model import copy_redist_src
+from attributecode.model import get_copy_list
 from attributecode.model import pre_process_and_fetch_license_dict
 from attributecode.model import write_output
 from attributecode.transform import transform_csv_to_csv
@@ -302,12 +303,16 @@ def gen(location, output, android, fetch_license, fetch_license_djc, reference,
     metavar='api_url api_key',
     help='Fetch licenses from a DejaCode License Library.')
 
+@click.option('--scancode',
+    is_flag=True,
+    help='Indicate the input JSON file is from scancode_toolkit.')
+
 @click.option('--verbose',
     is_flag=True,
     help='Show all error and warning messages.')
 
 @click.help_option('-h', '--help')
-def gen_license(location, output, djc, verbose):
+def gen_license(location, output, djc, scancode, verbose):
     """
 Fetch licenses in the license_expression field and save to the output location.
 
@@ -316,26 +321,25 @@ def gen_license(location, output, djc, verbose):
 OUTPUT: Path to a directory where license files are saved.
     """
     print_version()
+    api_url = ''
+    api_key = ''
+    errors = []
 
     if location.endswith('.csv') or location.endswith('.json') or location.endswith('.xlsx'):
-        _errors, abouts = load_inventory(
-            location=location
-        )
+        abouts = collect_inventory_license_expression(location=location, scancode=scancode)
     else:
-        _errors, abouts = collect_inventory(location)
-
+        #_errors, abouts = collect_inventory(location)
+        errors, abouts = collect_abouts_license_expression(location)
 
     log_file_loc = os.path.join(output, 'error.log')
-    api_url = ''
-    api_key = ''
-    errors = []
     if djc:
         # Strip the ' and " for api_url, and api_key from input
         api_url = djc[0].strip("'").strip('"')
         api_key = djc[1].strip("'").strip('"')
 
     click.echo('Fetching licenses...')
-    license_dict, lic_errors = pre_process_and_fetch_license_dict(abouts, api_url, api_key)
+    license_dict, lic_errors = pre_process_and_fetch_license_dict(abouts, api_url, api_key, scancode)
+
     if lic_errors:
         errors.extend(lic_errors)
 

src/attributecode/model.py

@@ -48,6 +48,7 @@
 from attributecode import api
 from attributecode import Error
 from attributecode import saneyaml
+from attributecode import gen
 from attributecode import util
 from attributecode.transform import write_excel
 from attributecode.util import add_unc
@@ -1325,6 +1326,60 @@ def collect_inventory(location):
     return unique(errors), abouts
 
 
+def collect_abouts_license_expression(location):
+    """
+    Read the ABOUT files at location and return a list of ABOUT objects without
+    validation. The purpose of this is to speed up the process for `gen_license` command.
+    """
+    lic_key_list = []
+    errors = []
+    input_location = util.get_absolute(location)
+    about_locations = list(util.get_about_locations(input_location))
+    abouts = []
+
+    for loc in about_locations:
+        try:
+            loc = add_unc(loc)
+            with io.open(loc, encoding='utf-8', errors='replace') as txt:
+                input_text = txt.read()
+            # saneyaml.load() will have parsing error if the input has
+            # tab value. Therefore, we should check if the input contains
+            # any tab and then convert it to spaces.
+            input = replace_tab_with_spaces(input_text)
+            data = saneyaml.load(input, allow_duplicate_keys=False)
+            about = About()
+            about.load_dict(data, base_dir='')
+            abouts.append(about)
+        except Exception as e:
+            trace = traceback.format_exc()
+            msg = 'Cannot load invalid ABOUT file: %(location)r: %(e)r\n%(trace)s'
+            errors.append(Error(CRITICAL, msg % locals()))
+
+    return errors, abouts
+
+
+def collect_inventory_license_expression(location, scancode=False):
+    """
+    Read the inventory file at location and return a list of  ABOUT objects without
+    validation. The purpose of this is to speed up the process for `gen_license` command.
+    """
+    abouts = []
+    if scancode:
+        inventory = gen.load_scancode_json(location)
+    else:
+        if location.endswith('.csv'):
+            inventory = gen.load_csv(location)
+        elif location.endswith('.xlsx'):
+            _dup_cols_err, inventory = gen.load_excel(location)
+        else:
+            inventory = gen.load_json(location)
+    for data in inventory:
+        about = About()
+        about.load_dict(data, base_dir='', scancode=scancode)
+        abouts.append(about)
+    return abouts
+
+
 def get_field_names(abouts):
     """
     Given a list of About objects, return a list of any field names that exist
@@ -1628,7 +1683,10 @@ def pre_process_and_fetch_license_dict(abouts, api_url=None, api_key=None, scanc
                                 lic_url = url + license_filename
                                 spdx_license_key = data['spdx_license_key']
                             except:
-                                msg = about.about_file_path + u" : Invalid 'license': " + lic_key
+                                try:
+                                    msg = about.about_file_path + u" : Invalid 'license': " + lic_key
+                                except:
+                                    msg = u"Invalid 'license': " + lic_key
                                 errors.append(Error(ERROR, msg))
                                 continue
                         detail_list.append(license_name)

src/attributecode/util.py

@@ -652,7 +652,6 @@ def load_scancode_json(location):
     """
     Read the scancode JSON file at `location` and return a list of dictionaries.
     """
-    mapping_dict = {}
     updated_results = []
 
     with open(location) as json_file:

tests/testdata/test_cmd/help/about_gen_license_help.txt

@@ -9,5 +9,6 @@ Usage: about gen-license [OPTIONS] LOCATION OUTPUT
 
 Options:
   --djc api_url api_key  Fetch licenses from a DejaCode License Library.
+  --scancode             Indicate the input JSON file is from scancode_toolkit.
   --verbose              Show all error and warning messages.
   -h, --help             Show this message and exit.