#518 - Add scancode option support for gen

Signed-off-by: Chin Yeung Li <[email protected]>

Author: chinyeungli

src/attributecode/cmd.py

@@ -224,6 +224,9 @@ def inventory(location, output, format, quiet, verbose):  # NOQA
               metavar='api_url api_key',
               help='Fetch license data and text files from a DejaCode License Library '
               'API URL using the API KEY.')
+@click.option('--scancode',
+              is_flag=True,
+              help='Indicate the input JSON file is from scancode_toolkit.')
 @click.option('--reference',
               metavar='DIR',
               type=click.Path(exists=True, file_okay=False,
@@ -239,7 +242,7 @@ def inventory(location, output, format, quiet, verbose):  # NOQA
               is_flag=True,
               help='Show all error and warning messages.')
 @click.help_option('-h', '--help')
-def gen(location, output, android, fetch_license, fetch_license_djc, reference, worksheet, quiet, verbose):
+def gen(location, output, android, fetch_license, fetch_license_djc, scancode, reference, worksheet, quiet, verbose):
     """
 Given a CSV/JSON/XLSX inventory, generate ABOUT files in the output location.
 
@@ -267,6 +270,7 @@ def gen(location, output, android, fetch_license, fetch_license_djc, reference,
         reference_dir=reference,
         fetch_license=fetch_license,
         fetch_license_djc=fetch_license_djc,
+        scancode=scancode,
         worksheet=worksheet
     )
 

src/attributecode/gen.py

@@ -97,7 +97,7 @@ def check_newline_in_file_field(component):
             try:
                 if '\n' in component[k]:
                     msg = ("New line character detected in '%s' for '%s' which is not supported."
-                            "\nPlease use ',' to declare multiple files.") % (k, component['about_resource'])
+                           "\nPlease use ',' to declare multiple files.") % (k, component['about_resource'])
                     errors.append(Error(CRITICAL, msg))
             except:
                 pass
@@ -111,7 +111,7 @@ def check_about_resource_filename(arp):
     """
     if invalid_chars(arp):
         msg = ("Invalid characters present in 'about_resource' "
-                   "field: " + arp)
+               "field: " + arp)
         return (Error(ERROR, msg))
     return ''
 
@@ -185,7 +185,8 @@ def load_inventory(location, from_attrib=False, base_dir=None, scancode=False, r
                 if from_attrib and f == 'about_resource':
                     continue
                 else:
-                    msg = "Required field: %(f)r not found in the <input>" % locals()
+                    msg = "Required field: %(f)r not found in the <input>" % locals(
+                    )
                     errors.append(Error(CRITICAL, msg))
                     return errors, abouts
         # Set about file path to '' if no 'about_resource' is provided from
@@ -238,19 +239,9 @@ def load_inventory(location, from_attrib=False, base_dir=None, scancode=False, r
 
         abouts.append(about)
     if custom_fields_list:
-        custom_fields_err_msg = 'Field ' + str(custom_fields_list) + ' is a custom field.'
+        custom_fields_err_msg = 'Field ' + \
+            str(custom_fields_list) + ' is a custom field.'
         errors.append(Error(INFO, custom_fields_err_msg))
-    # Covert the license_score value from string to list of int
-    # The licesne_score is not in the spec but is specify in the scancode license scan.
-    # This key will be treated as a custom string field. Therefore, we need to
-    # convert back to the list with float type for score.
-    if scancode:
-        for about in abouts:
-            try:
-                score_list = list(map(float, about.license_score.value.replace('[', '').replace(']', '').split(',')))
-                about.license_score.value = score_list
-            except:
-                pass
 
     return errors, abouts
 
@@ -259,7 +250,7 @@ def update_about_resource(self):
     pass
 
 
-def generate(location, base_dir, android=None, reference_dir=None, fetch_license=False, fetch_license_djc=False, worksheet=None):
+def generate(location, base_dir, android=None, reference_dir=None, fetch_license=False, fetch_license_djc=False, scancode=False, worksheet=None):
     """
     Load ABOUT data from a CSV inventory at `location`. Write ABOUT files to
     base_dir. Return errors and about objects.
@@ -287,11 +278,13 @@ def generate(location, base_dir, android=None, reference_dir=None, fetch_license
         location=location,
         base_dir=bdir,
         reference_dir=reference_dir,
+        scancode=scancode,
         worksheet=worksheet
     )
 
     if gen_license:
-        license_dict, err = model.pre_process_and_fetch_license_dict(abouts, api_url, api_key)
+        license_dict, err = model.pre_process_and_fetch_license_dict(
+            abouts, api_url, api_key)
         if err:
             for e in err:
                 # Avoid having same error multiple times
@@ -337,7 +330,8 @@ def generate(location, base_dir, android=None, reference_dir=None, fetch_license
                 # be validated when creating the about object
                 loc = util.to_posix(dump_loc)
                 about_file_loc = loc
-                path = join(dirname(util.to_posix(about_file_loc)), about_resource_value)
+                path = join(dirname(util.to_posix(about_file_loc)),
+                            about_resource_value)
                 if not exists(path):
                     path = util.to_posix(path.strip(UNC_PREFIX_POSIX))
                     path = normpath(path)
@@ -349,10 +343,12 @@ def generate(location, base_dir, android=None, reference_dir=None, fetch_license
             licenses_dict = {}
             if gen_license:
                 # Write generated LICENSE file
-                license_key_name_context_url_list = about.dump_lic(dump_loc, license_dict)
+                license_key_name_context_url_list = about.dump_lic(
+                    dump_loc, license_dict)
                 if license_key_name_context_url_list:
                     for lic_key, lic_name, lic_filename, lic_context, lic_url, spdx_lic_key in license_key_name_context_url_list:
-                        licenses_dict[lic_key] = [lic_name, lic_filename, lic_context, lic_url, spdx_lic_key]
+                        licenses_dict[lic_key] = [
+                            lic_name, lic_filename, lic_context, lic_url, spdx_lic_key]
                         if not lic_name in about.license_name.value:
                             about.license_name.value.append(lic_name)
                         about.license_file.value[lic_filename] = lic_filename

src/attributecode/model.py

@@ -1262,6 +1262,7 @@ def dumps(self, licenses_dict=None):
         # Handle license information that have not been handled.
         license_group = list(zip_longest(
             lic_key_copy, license_name, license_file, license_url, spdx_license_key))
+
         for lic_group in license_group:
             lic_dict = {}
             if lic_group[0]:

tests/test_cmd.py

@@ -43,7 +43,8 @@ def test_report_errors(capsys):
         Error(DEBUG, 'msg4'),
         Error(NOTSET, 'msg4'),
     ]
-    ec = cmd.report_errors(errors, quiet=False, verbose=True, log_file_loc=None)
+    ec = cmd.report_errors(errors, quiet=False,
+                           verbose=True, log_file_loc=None)
     assert 6 == ec
     out, err = capsys.readouterr()
     expected_out = [
@@ -67,7 +68,8 @@ def test_report_errors_without_verbose(capsys):
         Error(DEBUG, 'msg4'),
         Error(NOTSET, 'msg4'),
     ]
-    ec = cmd.report_errors(errors, quiet=False, verbose=False, log_file_loc=None)
+    ec = cmd.report_errors(errors, quiet=False,
+                           verbose=False, log_file_loc=None)
     assert 3 == ec
     out, err = capsys.readouterr()
     expected_out = [
@@ -153,7 +155,7 @@ def test_report_errors_can_write_to_logfile():
 
     result_file = get_temp_file()
     _ec = cmd.report_errors(errors, quiet=False, verbose=True,
-                           log_file_loc=result_file)
+                            log_file_loc=result_file)
     with open(result_file, 'r', encoding='utf-8', errors='replace') as rf:
         result = rf.read()
     expected = [
@@ -287,7 +289,7 @@ def test_parse_key_values_simple(self):
         expected = {
             'key': 'bar',
             'this': 'THat'
-            }
+        }
         keyvals, errors = cmd.parse_key_values(test)
         assert expected == keyvals
         assert not errors
@@ -332,6 +334,8 @@ def check_about_stdout(options, expected_loc, regen=False):
     with open(expected_file, 'r') as ef:
         expected = ef.read()
 
+    print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
+    print(result.output)
     assert expected.splitlines(False) == result.output.splitlines(False)
 
 
@@ -356,6 +360,7 @@ def test_about_gen_license_help_text():
         ['gen-license', '--help'],
         'test_cmd/help/about_gen_license_help.txt', regen=False)
 
+
 def test_about_check_help_text():
     check_about_stdout(
         ['check', '--help'],

tests/testdata/test_cmd/help/about_gen_help.txt

@@ -16,10 +16,12 @@ Options:
                                   Fetch license data and text files from a
                                   DejaCode License Library API URL using the API
                                   KEY.
+  --scancode                      Indicate the input JSON file is from
+                                  scancode_toolkit.
   --reference DIR                 Path to a directory with reference license
                                   data and text files.
   --worksheet name                The worksheet name from the INPUT. (Default:
                                   the "active" worksheet)
   -q, --quiet                     Do not print error or warning messages.
   --verbose                       Show all error and warning messages.
-  -h, --help                      Show this message and exit.
+  -h, --help                      Show this message and exit.