Fixed #482 - Support excel output for inventory

 * add encoding error handling (i.e. errors='replace')

Signed-off-by: Chin Yeung Li <[email protected]>

Author: chinyeungli

CHANGELOG.rst

@@ -13,8 +13,7 @@
     * Add Dockerfile to run aboutcode with docker
     * Add new option to choose extract license from ScanCode LicenseDB or DJC License Library
     * Add ability to transform Excel formatted file 
-    * Generate attribution directly from CSV/JSON/Excel file
-    * Generate ABOUT files with excel as an input
+    * Support Excel file format for `inventory`, `gen` and `attrib`
 
 2021-04-02
     Release 6.0.0

src/attributecode/cmd.py

@@ -139,7 +139,7 @@ def validate_extensions(ctx, param, value, extensions=tuple(('.csv', '.json',)))
 
 
 @about.command(cls=AboutCommand,
-    short_help='Collect the inventory of .ABOUT files to a CSV or JSON file.')
+    short_help='Collect the inventory of .ABOUT files to a CSV/JSON/Excel file.')
 
 @click.argument('location',
     required=True,
@@ -156,7 +156,7 @@ def validate_extensions(ctx, param, value, extensions=tuple(('.csv', '.json',)))
     is_flag=False,
     default='csv',
     show_default=True,
-    type=click.Choice(['json', 'csv']),
+    type=click.Choice(['json', 'csv', 'excel']),
     help='Set OUTPUT inventory file format.')
 
 @click.option('-q', '--quiet',
@@ -170,11 +170,11 @@ def validate_extensions(ctx, param, value, extensions=tuple(('.csv', '.json',)))
 @click.help_option('-h', '--help')
 def inventory(location, output, format, quiet, verbose):  # NOQA
     """
-Collect the inventory of ABOUT file data as CSV or JSON.
+Collect the inventory of .ABOUT files to a CSV/JSON/Excel file.
 
 LOCATION: Path to an ABOUT file or a directory with ABOUT files.
 
-OUTPUT: Path to the JSON or CSV inventory file to create.
+OUTPUT: Path to the CSV/JSON/Excel inventory file to create.
     """
     if not quiet:
         print_version()
@@ -184,8 +184,8 @@ def inventory(location, output, format, quiet, verbose):  # NOQA
         # accept zipped ABOUT files as input
         location = extract_zip(location)
     errors, abouts = collect_inventory(location)
-    write_errors = write_output(abouts=abouts, location=output, format=format)
-    errors.extend(write_errors)
+    write_output(abouts=abouts, location=output, format=format)
+
     errors = unique(errors)
     errors_count = report_errors(errors, quiet, verbose, log_file_loc=output + '-error.log')
     if not quiet:

src/attributecode/model.py

@@ -49,6 +49,7 @@
 from attributecode import Error
 from attributecode import saneyaml
 from attributecode import util
+from attributecode.transform import write_excel
 from attributecode.util import add_unc
 from attributecode.util import boolean_fields
 from attributecode.util import copy_license_notice_files
@@ -986,7 +987,7 @@ def load(self, location):
         errors = []
         try:
             loc = add_unc(loc)
-            with io.open(loc, encoding='utf-8') as txt:
+            with io.open(loc, encoding='utf-8', errors='replace') as txt:
                 input_text = txt.read()
             # The 'Yes' and 'No' will be converted to 'True' and 'False' in the yaml.load()
             # Therefore, we need to wrap the original value in quote to prevent
@@ -1513,34 +1514,28 @@ def write_output(abouts, location, format):  # NOQA
     about_dicts = about_object_to_list_of_dictionary(abouts)
     location = add_unc(location)
     if format == 'csv':
-        errors = save_as_csv(location, about_dicts, get_field_names(abouts))
+        save_as_csv(location, about_dicts, get_field_names(abouts))
+    elif format == 'json':
+        save_as_json(location, about_dicts)
     else:
-        errors = save_as_json(location, about_dicts)
-    return errors
-
+        save_as_excel(location, about_dicts)
 
 def save_as_json(location, about_dicts):
     with io.open(location, mode='w') as output_file:
         data = util.format_about_dict_for_json_output(about_dicts)
         output_file.write(json.dumps(data, indent=2))
-    return []
-
 
 def save_as_csv(location, about_dicts, field_names):
-    errors = []
     with io.open(location, mode='w', encoding='utf-8', newline='') as output_file:
         writer = csv.DictWriter(output_file, field_names)
         writer.writeheader()
-        csv_formatted_list = util.format_about_dict_for_csv_output(about_dicts)
+        csv_formatted_list = util.format_about_dict_output(about_dicts)
         for row in csv_formatted_list:
-            # See https://github.com/dejacode/about-code-tool/issues/167
-            try:
-                writer.writerow(row)
-            except Exception as e:
-                msg = u'Generation skipped for ' + row['about_file_path'] + u' : ' + str(e)
-                errors.append(Error(CRITICAL, msg))
-    return errors
+            writer.writerow(row)
 
+def save_as_excel(location, about_dicts):
+    formatted_list = util.format_about_dict_output(about_dicts)
+    write_excel(location, formatted_list)
 
 def pre_process_and_fetch_license_dict(abouts, api_url=None, api_key=None, scancode=False, reference=None):
     """

src/attributecode/util.py

@@ -486,8 +486,8 @@ def ungroup_licenses(licenses):
 
 
 # FIXME: add docstring
-def format_about_dict_for_csv_output(about_dictionary_list):
-    csv_formatted_list = []
+def format_about_dict_output(about_dictionary_list):
+    formatted_list = []
     for element in about_dictionary_list:
         row_list = dict()
         for key in element:
@@ -498,8 +498,8 @@ def format_about_dict_for_csv_output(about_dictionary_list):
                     row_list[key] = u'\n'.join((element[key].keys()))
                 else:
                     row_list[key] = element[key]
-        csv_formatted_list.append(row_list)
-    return csv_formatted_list
+        formatted_list.append(row_list)
+    return formatted_list
 
 
 # FIXME: add docstring

tests/test_model.py

@@ -850,14 +850,13 @@ def test_load_can_load_unicode(self):
         assert errors == a.errors
         assert 'Copyright (c) 2012, Domen Kožar' == a.copyright.value
 
-    def test_load_has_errors_for_non_unicode(self):
+    def test_load_non_unicode(self):
         test_file = get_test_loc('test_model/unicode/not-unicode.ABOUT')
         a = model.About()
         a.load(test_file)
         err = a.errors[0]
         assert CRITICAL == err.severity
         assert 'Cannot load invalid ABOUT file' in err.message
-        assert 'UnicodeDecodeError' in err.message
 
     def test_as_dict_load_dict_ignores_empties(self):
         test = {
@@ -1206,8 +1205,7 @@ def test_collect_inventory_does_not_convert_lf_to_crlf_from_directory(self):
         location = get_test_loc('test_model/crlf/about.ABOUT')
         result = get_temp_file()
         errors, abouts = model.collect_inventory(location)
-        errors2 = model.write_output(abouts, result, format='csv')
-        errors.extend(errors2)
+        model.write_output(abouts, result, format='csv')
         assert all(e.severity == INFO for e in errors)
 
         expected = get_test_loc('test_model/crlf/expected.csv')

tests/test_util.py

@@ -330,7 +330,7 @@ def test_load_csv_does_convert_column_names_to_lowercase(self):
         result = util.load_csv(test_file)
         assert expected == result
 
-    def test_format_about_dict_for_csv_output(self):
+    def test_format_about_dict_output(self):
         about = [dict([
             (u'about_file_path', u'/input/about1.ABOUT'),
             (u'about_resource', [u'test.c']),
@@ -345,7 +345,7 @@ def test_format_about_dict_for_csv_output(self):
             (u'license_expression', u'mit AND bsd-new'),
             (u'license_key', u'mit\nbsd-new')])]
 
-        output = util.format_about_dict_for_csv_output(about)
+        output = util.format_about_dict_output(about)
         assert output == expected
 
     def test_load_csv_microsoft_utf_8(self):

tests/testdata/test_cmd/help/about_help.txt

@@ -19,7 +19,7 @@ Commands:
                       report errors and warnings.
   collect-redist-src  Collect redistributable sources.
   gen                 Generate .ABOUT files from an inventory as CSV/JSON/Excel.
-  inventory           Collect the inventory of .ABOUT files to a CSV or JSON
+  inventory           Collect the inventory of .ABOUT files to a CSV/JSON/Excel
                       file.
   transform           Transform a CSV/JSON/Excel by applying renamings, filters
                       and checks.

tests/testdata/test_cmd/help/about_inventory_help.txt

@@ -1,13 +1,14 @@
 Usage: about inventory [OPTIONS] LOCATION OUTPUT
 
-  Collect the inventory of ABOUT file data as CSV or JSON.
+  Collect the inventory of .ABOUT files to a CSV/JSON/Excel file.
 
   LOCATION: Path to an ABOUT file or a directory with ABOUT files.
 
-  OUTPUT: Path to the JSON or CSV inventory file to create.
+  OUTPUT: Path to the CSV/JSON/Excel inventory file to create.
 
 Options:
-  -f, --format [json|csv]  Set OUTPUT inventory file format.  [default: csv]
-  -q, --quiet              Do not print error or warning messages.
-  --verbose                Show all error and warning messages.
-  -h, --help               Show this message and exit.
+  -f, --format [json|csv|excel]  Set OUTPUT inventory file format.  [default:
+                                 csv]
+  -q, --quiet                    Do not print error or warning messages.
+  --verbose                      Show all error and warning messages.
+  -h, --help                     Show this message and exit.