Fixed #523 - Add option to identify worksheet input

 * Able to identify worksheet name for the xlsx input for the tool to work with
 * Enhance error handling

Signed-off-by: Chin Yeung Li <[email protected]>

Author: chinyeungli

CHANGELOG.rst

@@ -5,6 +5,7 @@ Changelog
 
     * Fixed the transform code for xlsx and json
     * Remove irrelevant error for attrib
+    * Add support to identify worksheet name for XLSX input
 
 
 2022-10-24

about.ABOUT

@@ -1,6 +1,6 @@
 about_resource: .
 name: AboutCode-toolkit
-version: 7.2.1
+version: 7.3.0
 author: Jillian Daguil, Chin Yeung Li, Philippe Ombredanne, Thomas Druez
 copyright: Copyright (c) nexB Inc.
 description: |

docs/source/reference.rst

@@ -63,16 +63,19 @@ Options
                 --api_url URL                URL to DejaCode License Library.
                 --api_key KEY                API Key for the  DejaCode License Library
                 --min-license-score INTEGER  Attribute components that have license score
-                                             higher than the defined --min-license-score.
+                                            higher than or equal to the defined --min-
+                                            license-score.
                 --scancode                   Indicate the input JSON file is from
-                                             scancode toolkit.
+                                            scancode_toolkit.
                 --reference DIR              Path to a directory with reference files where
-                                             "license_file" and/or "notice_file" located.
-                --template FILE              Path to an optional custom attribution template to
-                                             generate the attribution document. If not provided
-                                             the default built-in template is used.
-                --vartext <key>=<value>      Add variable text as key=value for use in a custom
-                                             attribution template.
+                                            "license_file" and/or "notice_file" located.
+                --template FILE              Path to an optional custom attribution template
+                                            to generate the attribution document. If not
+                                            provided the default built-in template is used.
+                --vartext <key>=<value>      Add variable text as key=value for use in a
+                                            custom attribution template.
+                --worksheet name             The worksheet name from the INPUT. (Default:
+                                            the "active" worksheet)
                 -q, --quiet                  Do not print error or warning messages.
                 --verbose                    Show all error and warning messages.
                 -h, --help                   Show this message and exit.
@@ -149,6 +152,13 @@ Details
                     {{ vartext['title'] }}
                     {{ vartext['header'] }}
 
+                --worksheet
+
+                    This option identify the worksheet name from the XLSX input to work with.
+                    If no worksheet is defined, the "active" worksheet will be used
+
+                $ about attrib --worksheet BOM /home/project/audit.xlsx OUTPUT
+
                 --verbose
 
                     This option tells the tool to show all errors found.
@@ -326,28 +336,22 @@ Options
 
         ..  code-block:: none
 
-                --android                           Generate MODULE_LICENSE_XXX (XXX will be
-                                                    replaced by license key) and NOTICE as the same
-                                                    design as from Android.
-                --fetch-license                     Fetch license data and text files from the
-                                                    ScanCode LicenseDB.
-                --fetch-license-djc api_url api_key Fetch licenses data from DejaCode License
-                                                    Library and create <license>.LICENSE
-                                                    side-by-side with the generated .ABOUT file.
-                                                    The following additional options are required:
-
-                                                    api_url - URL to the DejaCode License Library
-                                                    API endpoint
-
-                                                    api_key - DejaCode API key
-                                                    Example syntax:
-
-                                                    about gen --fetch-license-djc api_url api_key
-                --reference PATH                    Path to a directory with reference license
-                                                    data and text files.
-                -q, --quiet                         Do not print any error/warning.
-                --verbose                           Show all the errors and warning.
-                -h, --help                          Show this message and exit.
+                --android                       Generate MODULE_LICENSE_XXX (XXX will be
+                                                replaced by license key) and NOTICE as the
+                                                same design as from Android.
+                --fetch-license                 Fetch license data and text files from the
+                                                ScanCode LicenseDB.
+                --fetch-license-djc api_url api_key
+                                                Fetch license data and text files from a
+                                                DejaCode License Library API URL using the
+                                                API KEY.
+                --reference DIR                 Path to a directory with reference license
+                                                data and text files.
+                --worksheet name                The worksheet name from the INPUT. (Default:
+                                                the "active" worksheet)
+                -q, --quiet                     Do not print error or warning messages.
+                --verbose                       Show all error and warning messages.
+                -h, --help                      Show this message and exit.
 
 Purpose
 -------
@@ -405,6 +409,13 @@ Details
 
                 $ about gen --reference /home/licenses_notices/ LOCATION OUTPUT
 
+                --worksheet
+
+                    This option identify the worksheet name from the XLSX input to work with.
+                    If no worksheet is defined, the "active" worksheet will be used
+
+                $ about gen --worksheet BOM LOCATION OUTPUT
+
                 --verbose
 
                     This option tells the tool to show all errors found.
@@ -428,13 +439,13 @@ Options
 
         ..  code-block:: none
 
-                --djc api_url api_key   Fetch licenses data from DejaCode License
-                                        Library and create <license>.LICENSE to the
-                                        OUTPUT location.
-                --scancode              Indicate the input JSON file is from
+                --djc api_url api_key  Fetch licenses from a DejaCode License Library.
+                --scancode             Indicate the input JSON file is from
                                         scancode_toolkit.
-                 --verbose              Show all the errors and warning.
-                -h, --help              Show this message and exit.
+                --worksheet name       The worksheet name from the INPUT. (Default: the
+                                        "active" worksheet)
+                --verbose              Show all error and warning messages.
+                -h, --help             Show this message and exit.
 
 Purpose
 -------
@@ -467,6 +478,13 @@ Details
 
                 $ about gen_license --scancode /home/project/scancode-license-detection.json OUTPUT
 
+                --worksheet
+
+                    This option identify the worksheet name from the XLSX input to work with.
+                    If no worksheet is defined, the "active" worksheet will be used
+
+                $ about gen_license --worksheet BOM /home/project/bom-v0.10.xlsx OUTPUT
+
                 --verbose
 
                     This option tells the tool to show all errors found.
@@ -594,7 +612,9 @@ Options
         ..  code-block:: none
 
                 -c, --configuration FILE  Path to an optional YAML configuration file. See
-                                          --help-format for format help.
+                                            --help-format for format help.
+                --worksheet name          The worksheet name from the INPUT. (Default: the
+                                            "active" worksheet)
                 --help-format             Show configuration file format help and exit.
                 -q, --quiet               Do not print error or warning messages.
                 --verbose                 Show all error and warning messages.
@@ -617,6 +637,13 @@ Details
 
                 $ about transform -c 'path to the YAML configuration file' LOCATION OUTPUT
 
+                --worksheet
+
+                    This option identify the worksheet name from the XLSX input to work with.
+                    If no worksheet is defined, the "active" worksheet will be used
+
+                $ about transform -c 'path to the YAML configuration file' --worksheet BOM /project/bom-v.20.xlsx OUTPUT
+
                 --help-format
 
                     Show configuration file format help and exit.

src/attributecode/cmd.py

@@ -239,6 +239,10 @@ def inventory(location, output, format, quiet, verbose):  # NOQA
     type=click.Path(exists=True, file_okay=False, readable=True, resolve_path=True),
     help='Path to a directory with reference license data and text files.')
 
+@click.option('--worksheet',
+    metavar='name',
+    help='The worksheet name from the INPUT. (Default: the "active" worksheet)')
+
 @click.option('-q', '--quiet',
     is_flag=True,
     help='Do not print error or warning messages.')
@@ -248,7 +252,7 @@ def inventory(location, output, format, quiet, verbose):  # NOQA
     help='Show all error and warning messages.')
 
 @click.help_option('-h', '--help')
-def gen(location, output, android, fetch_license, fetch_license_djc, reference, quiet, verbose):
+def gen(location, output, android, fetch_license, fetch_license_djc, reference, worksheet, quiet, verbose):
     """
 Given a CSV/JSON/XLSX inventory, generate ABOUT files in the output location.
 
@@ -264,13 +268,17 @@ def gen(location, output, android, fetch_license, fetch_license_djc, reference,
     if not location.endswith(('.csv', '.json', '.xlsx')):
         raise click.UsageError('ERROR: Invalid input file extension: must be one .csv or .json or .xlsx.')
 
+    if worksheet and not location.endswith('.xlsx'):
+        raise click.UsageError('ERROR: --worksheet option only works with .xlsx input.')
+
     errors, abouts = generate_about_files(
         location=location,
         base_dir=output,
         android=android,
         reference_dir=reference,
         fetch_license=fetch_license,
         fetch_license_djc=fetch_license_djc,
+        worksheet=worksheet
     )
 
     errors_count = report_errors(errors, quiet, verbose, log_file_loc=output + '-error.log')
@@ -309,12 +317,16 @@ def gen(location, output, android, fetch_license, fetch_license_djc, reference,
     is_flag=True,
     help='Indicate the input JSON file is from scancode_toolkit.')
 
+@click.option('--worksheet',
+    metavar='name',
+    help='The worksheet name from the INPUT. (Default: the "active" worksheet)')
+
 @click.option('--verbose',
     is_flag=True,
     help='Show all error and warning messages.')
 
 @click.help_option('-h', '--help')
-def gen_license(location, output, djc, scancode, verbose):
+def gen_license(location, output, djc, scancode, worksheet, verbose):
     """
 Fetch licenses (Default: ScanCode LicenseDB) in the license_expression field and save to the output location.
 
@@ -327,10 +339,13 @@ def gen_license(location, output, djc, scancode, verbose):
     api_key = ''
     errors = []
 
+    if worksheet and not location.endswith('.xlsx'):
+        raise click.UsageError('ERROR: --worksheet option only works with .xlsx input.')
+
     log_file_loc = os.path.join(output, 'error.log')
 
     if location.endswith('.csv') or location.endswith('.json') or location.endswith('.xlsx'):
-        errors, abouts = collect_inventory_license_expression(location=location, scancode=scancode)
+        errors, abouts = collect_inventory_license_expression(location=location, scancode=scancode, worksheet=worksheet)
         if errors:
             severe_errors_count = report_errors(errors, quiet=False, verbose=verbose, log_file_loc=log_file_loc)
             sys.exit(severe_errors_count)
@@ -440,6 +455,10 @@ def validate_template(ctx, param, value):
     metavar='<key>=<value>',
     help='Add variable text as key=value for use in a custom attribution template.')
 
+@click.option('--worksheet',
+    metavar='name',
+    help='The worksheet name from the INPUT. (Default: the "active" worksheet)')
+
 @click.option('-q', '--quiet',
     is_flag=True,
     help='Do not print error or warning messages.')
@@ -449,7 +468,7 @@ def validate_template(ctx, param, value):
     help='Show all error and warning messages.')
 
 @click.help_option('-h', '--help')
-def attrib(input, output, api_url, api_key, scancode, min_license_score, reference, template, vartext, quiet, verbose):
+def attrib(input, output, api_url, api_key, scancode, min_license_score, reference, template, vartext, worksheet, quiet, verbose):
     """
 Generate an attribution document at OUTPUT using JSON, CSV or XLSX or .ABOUT files at INPUT.
 
@@ -464,6 +483,9 @@ def attrib(input, output, api_url, api_key, scancode, min_license_score, referen
     license_dict = {}
     errors = []
 
+    if worksheet and not input.endswith('.xlsx'):
+        raise click.UsageError('ERROR: --worksheet option only works with .xlsx input.')
+
     if not quiet:
         print_version()
         click.echo('Generating attribution...')
@@ -501,7 +523,8 @@ def attrib(input, output, api_url, api_key, scancode, min_license_score, referen
             location=input,
             from_attrib=from_attrib,
             scancode=scancode,
-            reference_dir=reference
+            reference_dir=reference,
+            worksheet=worksheet
         )
 
     else:
@@ -765,6 +788,10 @@ def print_config_help(ctx, param, value):
     help='Path to an optional YAML configuration file. See --help-format for '
          'format help.')
 
+@click.option('--worksheet',
+    metavar='name',
+    help='The worksheet name from the INPUT. (Default: the "active" worksheet)')
+
 @click.option('--help-format',
     is_flag=True, is_eager=True, expose_value=False,
     callback=print_config_help,
@@ -779,7 +806,7 @@ def print_config_help(ctx, param, value):
     help='Show all error and warning messages.')
 
 @click.help_option('-h', '--help')
-def transform(location, output, configuration, quiet, verbose):  # NOQA
+def transform(location, output, configuration, worksheet, quiet, verbose):  # NOQA
     """
 Transform the CSV/JSON/XLSX file at LOCATION by applying renamings, filters and checks
 and then write a new CSV/JSON/XLSX to OUTPUT.
@@ -788,6 +815,9 @@ def transform(location, output, configuration, quiet, verbose):  # NOQA
 
 OUTPUT: Path to CSV/JSON/XLSX inventory file to create.
     """
+    if worksheet and not location.endswith('.xlsx'):
+        raise click.UsageError('ERROR: --worksheet option only works with .xlsx input.')
+
     if not configuration:
         transformer = Transformer.default()
     else:
@@ -807,11 +837,16 @@ def transform(location, output, configuration, quiet, verbose):  # NOQA
     elif location.endswith('.json'):
         new_data, errors = transform_json(location)
     elif location.endswith('.xlsx'):
-        new_data, errors = transform_excel(location)
+        new_data, errors = transform_excel(location, worksheet)
 
     if not errors:
         updated_data, errors = transform_data(new_data, transformer)
 
+    if not updated_data:
+        msg = 'The input is empty. Nothing is transformed.'
+        click.echo(msg)
+        sys.exit(0)
+
     if not errors:
         if output.endswith('.csv'):
             write_csv(output, updated_data)

src/attributecode/gen.py

@@ -116,7 +116,7 @@ def check_about_resource_filename(arp):
     return ''
 
 
-def load_inventory(location, from_attrib=False, base_dir=None, scancode=False, reference_dir=None):
+def load_inventory(location, from_attrib=False, base_dir=None, scancode=False, reference_dir=None, worksheet=None):
     """
     Load the inventory file at `location` for ABOUT and LICENSE files stored in
     the `base_dir`. Return a list of errors and a list of About objects
@@ -140,7 +140,7 @@ def load_inventory(location, from_attrib=False, base_dir=None, scancode=False, r
                 return errors, abouts
             inventory = load_csv(location)
         elif location.endswith('.xlsx'):
-            dup_cols_err, inventory = load_excel(location)
+            dup_cols_err, inventory = load_excel(location, worksheet)
             if dup_cols_err:
                 errors.extend(dup_cols_err)
                 return errors, abouts
@@ -266,7 +266,7 @@ def update_about_resource(self):
     pass
 
 
-def generate(location, base_dir, android=None, reference_dir=None, fetch_license=False, fetch_license_djc=False):
+def generate(location, base_dir, android=None, reference_dir=None, fetch_license=False, fetch_license_djc=False, worksheet=None):
     """
     Load ABOUT data from a CSV inventory at `location`. Write ABOUT files to
     base_dir. Return errors and about objects.
@@ -293,7 +293,8 @@ def generate(location, base_dir, android=None, reference_dir=None, fetch_license
     errors, abouts = load_inventory(
         location=location,
         base_dir=bdir,
-        reference_dir=reference_dir
+        reference_dir=reference_dir,
+        worksheet=worksheet
     )
 
     if gen_license:

src/attributecode/model.py

@@ -1392,7 +1392,7 @@ def collect_abouts_license_expression(location):
     return errors, abouts
 
 
-def collect_inventory_license_expression(location, scancode=False):
+def collect_inventory_license_expression(location, scancode=False, worksheet=None):
     """
     Read the inventory file at location and return a list of  ABOUT objects without
     validation. The purpose of this is to speed up the process for `gen_license` command.
@@ -1410,11 +1410,11 @@ def collect_inventory_license_expression(location, scancode=False):
         if location.endswith('.csv'):
             inventory = gen.load_csv(location)
         elif location.endswith('.xlsx'):
-            _dup_cols_err, inventory = gen.load_excel(location)
+            _dup_cols_err, inventory = gen.load_excel(location, worksheet)
         else:
             inventory = gen.load_json(location)
         # Check if 'license_expression' field is in the input
-        if not 'license_expression' in inventory[0]:
+        if not inventory or not 'license_expression' in inventory[0]:
             errors.append(Error(CRITICAL, "No 'license_expression' field in the input."))
             return errors, abouts