Option for --validate-about-resource #300

 * Add the `--validate-about-resource` options for both `gen`,
`inventory` and `attrib`
 * The code will still do the validation, but it will not show the error
if `--validate-about-resource` option is not used.
 * Update USAGE doc

Signed-off-by: Chin Yeung Li <[email protected]>

Author: chinyeungli

USAGE.rst

@@ -40,12 +40,14 @@ attrib
 
 ::
 
-    --inventory PATH  Path to an inventory file
-    --mapping         Use for mapping between the input keys and the ABOUT field
-                      names - mapping.config
-    --template PATH   Path to a custom attribution template
-    -q, --quiet       Do not print any error/warning.
-    --help            Show this message and exit.
+    --inventory PATH            Path to an inventory file.
+    --mapping                   Use for mapping between the input keys and the ABOUT field.
+                                names - mapping.config
+    --show-all                  Show all the errors and warning
+    --template PATH             Path to a custom attribution template.
+    --validate-about-resource   Validate the existence of the about resource.
+    -q, --quiet                 Do not print any error/warning.
+    --help                      Show this message and exit.
 
 Purpose
 -------
@@ -81,12 +83,21 @@ Options
 
         See mapping.config for details
 
+    --show-all
+
+        This option ask the tool to show all kind of errors found.
+        The default behavior will only show 'CRITICAL', 'ERROR', and 'WARNING'
+
     --template
 
         This option allows users to use their own template for attribution generation.
         For instance, if user has a custom template located at:
         /home/custom_template/template.html
 
+    --validate-about-resource
+
+        The option ask the tool to validate the existence of the referencing file.
+
     $ about attrib --template /home/custom_template/template.html LOCATION OUTPUT
 
 
@@ -166,6 +177,8 @@ gen
                                         the generated location
     --mapping                           Use for mapping between the input keys and
                                         the ABOUT field names - mapping.config
+    --show-all                          Show all the errors and warning
+    --validate-about-resource           Validate the existence of the about resource.
     -q, --quiet                         Do not print any error/warning.
     --help                              Show this message and exit.
 
@@ -206,6 +219,14 @@ Options
 
         See mapping.config for details
 
+    --show-all
+
+        This option ask the tool to show all kind of errors found.
+        The default behavior will only show 'CRITICAL', 'ERROR', and 'WARNING'
+
+    --validate-about-resource
+
+        The option ask the tool to validate the existence of the referencing file.
 
 inventory
 =========
@@ -223,9 +244,11 @@ inventory
 
 ::
 
-    -f, --format [json|csv]  Set OUTPUT file format.  [default: csv]
-    -q, --quiet              Do not print any error/warning.
-    --help                   Show this message and exit.
+    -f, --format [json|csv]     Set OUTPUT file format.  [default: csv]
+    --show-all                  Show all the errors and warning
+    --validate-about-resource   Validate the existence of the about resource.
+    -q, --quiet                 Do not print any error/warning.
+    --help                      Show this message and exit.
 
 Purpose
 -------
@@ -240,4 +263,13 @@ Options
  
         Set OUTPUT file format.  [default: csv]
 
+    --show-all
+
+        This option ask the tool to show all kind of errors found.
+        The default behavior will only show 'CRITICAL', 'ERROR', and 'WARNING'
+
+    --validate-about-resource
+
+        The option ask the tool to validate the existence of the referencing file.
+
     $ about inventory -f json [OPTIONS] LOCATION OUTPUT

src/attributecode/cmd.py

@@ -35,6 +35,7 @@
 from attributecode import model
 from attributecode import severities
 from attributecode.util import extract_zip
+from attributecode.util import ignore_about_resource_path_error
 from attributecode.util import to_posix
 
 
@@ -119,12 +120,16 @@ def cli():
         'for any level.'
 )
 
+@click.option('--validate-about-resource', is_flag=True, default=False,
+    help='Validate the existence of the about resource.'
+)
+
 @click.option('-q', '--quiet', is_flag=True,
     help='Do not print error or warning messages.')
 
 @click.help_option('-h', '--help')
 
-def inventory(location, output, quiet, format, show_all):
+def inventory(location, output, quiet, format, show_all, validate_about_resource):
     """
 Collect a JSON or CSV inventory of components from .ABOUT files.
 
@@ -152,7 +157,15 @@ def inventory(location, output, quiet, format, show_all):
     write_errors = model.write_output(abouts, output, format)
     for err in write_errors:
         errors.append(err)
-        log_errors(errors, quiet, show_all, os.path.dirname(output))
+
+    finalized_errors = []
+
+    if not validate_about_resource:
+        finalized_errors = ignore_about_resource_path_error(errors)
+    else:
+        finalized_errors = errors
+    
+    log_errors(finalized_errors, quiet, show_all, os.path.dirname(output))
     sys.exit(0)
 
 
@@ -183,10 +196,10 @@ def inventory(location, output, quiet, format, show_all):
 # TODO: this option help and long name is obscure and would need to be refactored
 @click.option('--license-notice-text-location', nargs=1,
     type=click.Path(exists=True, dir_okay=True, readable=True, resolve_path=True),
-    help="Copy the 'license_file' from the directory to the generated location")
+    help="Copy the 'license_file' from the directory to the generated location.")
 
 @click.option('--mapping', is_flag=True,
-    help='Use file mapping.config with mapping between input keys and ABOUT field names')
+    help='Use file mapping.config with mapping between input keys and ABOUT field names.')
 
 @click.option('--show-all', is_flag=True, default=False,
     help='Show all errors and warnings. '
@@ -223,20 +236,25 @@ def gen(location, output, mapping, license_notice_text_location, fetch_license,
     click.echo('Generating .ABOUT files...')
 
     errors, abouts = gen_generate(
-        location=location, base_dir=output, validate_about_resource=validate_about_resource,
-        license_notice_text_location=license_notice_text_location,
+        location=location, base_dir=output, license_notice_text_location=license_notice_text_location,
         fetch_license=fetch_license, use_mapping=mapping)
 
     about_count = len(abouts)
     error_count = 0
+    finalized_errors = []
+
+    if not validate_about_resource:
+        finalized_errors = ignore_about_resource_path_error(errors)
+    else:
+        finalized_errors = errors
 
-    for e in errors:
+    for e in finalized_errors:
         # Only count as warning/error if CRITICAL, ERROR and WARNING
         if e.severity > 20:
             error_count = error_count + 1
     click.echo(
         'Generated %(about_count)d .ABOUT files with %(error_count)d errors or warnings' % locals())
-    log_errors(errors, quiet, show_all, output)
+    log_errors(finalized_errors, quiet, show_all, output)
     sys.exit(0)
 
 
@@ -256,12 +274,12 @@ def gen(location, output, mapping, license_notice_text_location, fetch_license,
 @click.option('--inventory', required=False,
     type=click.Path(exists=True, file_okay=True, resolve_path=True),
     help='Path to an optional JSON or CSV inventory file listing the '
-        'subset of .ABOUT files path to consider when generating attribution '
+        'subset of .ABOUT files path to consider when generating attribution.'
     )
 
 @click.option('--mapping', is_flag=True,
     help='Use the file "mapping.config" with mappings between the CSV '
-        'inventory columns names and .ABOUT field names')
+        'inventory columns names and .ABOUT field names.')
 
 @click.option('--show-all', is_flag=True, default=False,
     help='Show all errors and warnings. '
@@ -274,12 +292,17 @@ def gen(location, output, mapping, license_notice_text_location, fetch_license,
 @click.option('--template', type=click.Path(exists=True), nargs=1,
     help='Path to an optional custom attribution template used for generation.')
 
+@click.option('--validate-about-resource', is_flag=True, default=False,
+    help='Validate the existence of the about resource.'
+)
+
 @click.option('-q', '--quiet', is_flag=True,
     help='Do not print error or warning messages.')
 
 @click.help_option('-h', '--help')
 
-def attrib(location, output, template, mapping, inventory, quiet, show_all):
+def attrib(location, output, template, mapping, inventory, quiet, show_all,
+           validate_about_resource):
     """
 Generate an attribution document at OUTPUT using .ABOUT files at LOCATION.
 
@@ -303,7 +326,14 @@ def attrib(location, output, template, mapping, inventory, quiet, show_all):
     for no_match_error in no_match_errors:
         inv_errors.append(no_match_error)
 
-    log_errors(inv_errors, quiet, show_all, os.path.dirname(output))
+    finalized_errors = []
+
+    if not validate_about_resource:
+        finalized_errors = ignore_about_resource_path_error(inv_errors)
+    else:
+        finalized_errors = inv_errors
+
+    log_errors(finalized_errors, quiet, show_all, os.path.dirname(output))
     click.echo('Finished.')
     sys.exit(0)
 

src/attributecode/gen.py

@@ -178,10 +178,9 @@ def load_inventory(location, base_dir, license_notice_text_location=None,
     return errors, abouts
 
 
-def generate(location, base_dir, validate_about_resource=False,
-             license_notice_text_location=None, fetch_license=False, policy=None,
-             conf_location=None, with_empty=False, with_absent=False,
-             use_mapping=False):
+def generate(location, base_dir, license_notice_text_location=None,
+             fetch_license=False, policy=None, conf_location=None,
+             with_empty=False, with_absent=False, use_mapping=False):
     """
     Load ABOUT data from a CSV inventory at `location`. Write ABOUT files to
     base_dir using policy flags and configuration file at conf_location.
@@ -256,22 +255,21 @@ def generate(location, base_dir, validate_about_resource=False,
                 for about_resource_value in about.about_resource.value:
                     about.about_resource_path.value[about_resource_value] = None
                 about.about_resource_path.present = True
-                if validate_about_resource:
-                    # Check for the existence of the about_resource
-                    # If the input already have the about_resource_path field, it will
-                    # be validated when creating the about object
-                    loc = util.to_posix(dump_loc)
-                    about_file_loc = loc
-                    for about_resource_value in about.about_resource_path.value:
-                        path = join(dirname(util.to_posix(about_file_loc)),
-                                              about_resource_value)
-                        if not exists(path):
-                            path = util.to_posix(path.strip(UNC_PREFIX_POSIX))
-                            path = normpath(path)
-                            msg = (u'The reference file: '
-                                   u'%(path)s '
-                                   u'does not exist' % locals())
-                            not_exist_errors.append(msg)
+                # Check for the existence of the about_resource
+                # If the input already have the about_resource_path field, it will
+                # be validated when creating the about object
+                loc = util.to_posix(dump_loc)
+                about_file_loc = loc
+                for about_resource_value in about.about_resource_path.value:
+                    path = join(dirname(util.to_posix(about_file_loc)),
+                                          about_resource_value)
+                    if not exists(path):
+                        path = util.to_posix(path.strip(UNC_PREFIX_POSIX))
+                        path = normpath(path)
+                        msg = (u'Field about_resource_path: '
+                               u'%(path)s '
+                               u'does not exist' % locals())
+                        not_exist_errors.append(msg)
             if gen_license:
                 about.license_file.value = OrderedDict()
                 # Write generated LICENSE file

src/attributecode/util.py

@@ -471,3 +471,14 @@ def copy_license_notice_files(fields, base_dir, license_notice_text_location, af
             if not posixpath.exists(to_lic_path):
                 os.makedirs(to_lic_path)
             shutil.copy2(from_lic_path, to_lic_path)
+
+
+def ignore_about_resource_path_error(errors):
+    ignore_resource_path_check_message = u'Field about_resource_path:'
+    updated_errors = []
+    for err in errors:
+        if ignore_resource_path_check_message in err.message:
+            continue
+        else:
+            updated_errors.append(err)
+    return updated_errors

tests/test_gen.py

@@ -113,40 +113,37 @@ def test_generation_with_no_about_resource(self):
         errors, abouts = gen.generate(location, base_dir)
         expected = [u'.']
         assert abouts[0].about_resource.value == expected
-        assert len(errors) == 0
+        assert len(errors) == 1
 
     def test_generation_with_no_about_resource_reference(self):
         location = get_test_loc('gen/inv3.csv')
         base_dir = get_temp_dir()
-        validate_about_resource = True
 
-        errors, abouts = gen.generate(location, base_dir, validate_about_resource)
+        errors, abouts = gen.generate(location, base_dir)
         expected = [u'test.tar.gz']
 
         assert abouts[0].about_resource.value == expected
         assert len(errors) == 1
-        msg = u'The reference file'
+        msg = u'Field about_resource_path'
         assert msg in errors[0].message
 
     def test_generation_with_no_about_resource_reference_no_resource_validation(self):
         location = get_test_loc('gen/inv3.csv')
         base_dir = get_temp_dir()
-        validate_about_resource = False
 
-        errors, abouts = gen.generate(location, base_dir, validate_about_resource)
+        errors, abouts = gen.generate(location, base_dir)
         expected = [u'test.tar.gz']
 
         assert abouts[0].about_resource.value == expected
-        assert len(errors) == 0
+        assert len(errors) == 1
 
     def test_generate(self):
         location = get_test_loc('gen/inv.csv')
         base_dir = get_temp_dir()
-        validate_about_resource = True
 
-        errors, abouts = gen.generate(location, base_dir, validate_about_resource)
+        errors, abouts = gen.generate(location, base_dir)
         msg1 = u'Field custom1 is not a supported field and is ignored.'
-        msg2 = u'The reference file'
+        msg2 = u'Field about_resource_path'
 
         assert msg1 in errors[0].message
         assert msg2 in errors[1].message

tests/test_util.py

@@ -29,6 +29,7 @@
 from testing_utils import on_windows
 
 from attributecode import CRITICAL
+from attributecode import ERROR
 from attributecode import Error
 from attributecode import util
 
@@ -338,3 +339,8 @@ def test_apply_mapping(self):
                          ])]
         assert util.apply_mapping(input) == expected
 
+    def test_ignore_about_resource_path_error(self):
+        input_err = [Error(ERROR, 'Field about_resource_path: test.tar.gz does not exist'),
+                     Error(ERROR, 'Field about_resource_path: test.tar.gz does not exist')]
+        expected_err = []
+        assert util.ignore_about_resource_path_error(input_err) == expected_err