Merge pull request #375 from nexB/361-remove-mappings

Remove mappings #361

Author: pombredanne

src/attributecode/__init__.py

@@ -65,13 +65,24 @@ def __new__(self, severity, message):
             Error, severity, message)
 
     def __repr__(self, *args, **kwargs):
+        sev, msg = self._get_values()
+        return 'Error(%(sev)s,  %(msg)s)' % locals()
+
+    def __eq__(self, other):
+        return repr(self) == repr(other)
+
+    def _get_values(self):
         sev = severities[self.severity]
         msg = self._clean_string(repr(self.message))
-        return 'Error(%(sev)s, %(msg)s)' % locals()
+        return sev, msg
+
+    def render(self):
+        sev, msg = self._get_values()
+        return '%(sev)s: %(msg)s' % locals()
 
     def to_dict(self, *args, **kwargs):
         """
-        Return an ordered mapping of self.
+        Return an ordered dict of self.
         """
         return self._asdict()
 
@@ -114,7 +125,3 @@ def _clean_string(s):
     DEBUG : 'DEBUG',
     NOTSET : 'NOTSET'
 }
-
-
-DEFAULT_MAPPING = os.path.join(os.path.abspath(
-    os.path.dirname(__file__)), 'mapping.config')

src/attributecode/__main__.py

@@ -19,6 +19,6 @@
 from __future__ import unicode_literals
 
 
-if __name__ == '__main__': # pragma: nocover
+if __name__ == '__main__':  # pragma: nocover
     from attributecode import cmd
-    cmd.cli()
+    cmd.about()

src/attributecode/attrib.py

@@ -41,7 +41,7 @@
 def generate(abouts, template=None, variables=None):
     """
     Generate an attribution text from an `abouts` list of About objects, a
-    `template` template text and a `variables` optional mapping of extra
+    `template` template text and a `variables` optional dict of extra
     variables.
 
     Return a tuple of (error, attribution text) where error is an Error object
@@ -153,28 +153,25 @@ def generate_from_file(abouts, template_loc=DEFAULT_TEMPLATE_FILE, variables=Non
     """
     Generate an attribution text from an `abouts` list of About objects, a
     `template_loc` template file location and a `variables` optional
-    mapping of extra variables.
+    dict of extra variables.
 
     Return a tuple of (error, attribution text) where error is an Error object
     or None and attribution text is the generated text or None.
     """
-    
+
     template_loc = add_unc(template_loc)
     with io.open(template_loc, encoding='utf-8') as tplf:
         tpls = tplf.read()
     return generate(abouts, template=tpls, variables=variables)
 
 
-def generate_and_save(abouts, output_location, template_loc=None, variables=None,
-                      mapping_file=None):
+def generate_and_save(abouts, output_location, template_loc=None, variables=None):
     """
     Generate an attribution text from an `abouts` list of About objects, a
     `template_loc` template file location and a `variables` optional
-    mapping of extra variables. Save the generated attribution text in the
-    `output_location` file. 
+    dict of extra variables. Save the generated attribution text in the
+    `output_location` file.
     Return a list of Error objects if any.
-
-    Optionally use the `mapping_file` mapping config if provided.
     """
     updated_abouts = []
     errors = []
@@ -200,7 +197,7 @@ def generate_and_save(abouts, output_location, template_loc=None, variables=None
     if rendering_error:
         errors.append(rendering_error)
 
-    if rendered:    
+    if rendered:
         output_location = add_unc(output_location)
         with io.open(output_location, 'w', encoding='utf-8') as of:
             of.write(rendered)

src/attributecode/cmd.py

@@ -34,7 +34,6 @@
 
 from attributecode import __about_spec_version__
 from attributecode import __version__
-from attributecode import DEFAULT_MAPPING
 from attributecode import severities
 from attributecode.attrib import check_template
 from attributecode.attrib import DEFAULT_TEMPLATE_FILE
@@ -43,6 +42,7 @@
 from attributecode.model import collect_inventory
 from attributecode.model import write_output
 from attributecode.util import extract_zip
+from attributecode.util import filter_errors
 
 
 __copyright__ = """
@@ -107,7 +107,7 @@ def about():
 
 def validate_key_values(ctx, param, value):
     """
-    Return the a mapping of {key: [values,...] if valid or raise a UsageError
+    Return the a dict of {key: [values,...] if valid or raise a UsageError
     otherwise.
     """
     if not value:
@@ -122,20 +122,6 @@ def validate_key_values(ctx, param, value):
     return kvals
 
 
-def validate_mapping(mapping, mapping_file):
-    """
-    Return a mapping_file or None.
-    Raise a UsageError on errors.
-    """
-    if mapping and mapping_file:
-        raise click.UsageError(
-            'Invalid options combination: '
-            '--mapping and --mapping-file are mutually exclusive.')
-    if mapping:
-        return DEFAULT_MAPPING
-    return mapping_file or None
-
-
 def validate_extensions(ctx, param, value, extensions=tuple(('.csv', '.json',))):
     if not value:
         return
@@ -171,19 +157,6 @@ def validate_extensions(ctx, param, value, extensions=tuple(('.csv', '.json',)))
     type=click.Choice(['json', 'csv']),
     help='Set OUTPUT inventory file format.')
 
-@click.option('--mapping',
-    is_flag=True,
-    help='Use the default built-in "mapping.config" file '
-         'with mapping between input keys and .ABOUT field names.'
-         'Cannot be combined with the --mapping-file option.')
-
-@click.option('--mapping-file',
-    metavar='FILE',
-    type=click.Path(exists=True, dir_okay=False, readable=True, resolve_path=True),
-    help='Path to an optional custom mapping FILE '
-         'with mapping between input keys and .ABOUT field names. '
-         'Cannot be combined with the --mapping option.')
-
 @click.option('-q', '--quiet',
     is_flag=True,
     help='Do not print error or warning messages.')
@@ -194,8 +167,7 @@ def validate_extensions(ctx, param, value, extensions=tuple(('.csv', '.json',)))
 
 @click.help_option('-h', '--help')
 
-def inventory(location, output, mapping, mapping_file,
-              format, quiet, verbose):  # NOQA
+def inventory(location, output, format, quiet, verbose):  # NOQA
     """
 Collect the inventory of .ABOUT file data as CSV or JSON.
 
@@ -211,29 +183,9 @@ def inventory(location, output, mapping, mapping_file,
     if location.lower().endswith('.zip'):
         # accept zipped ABOUT files as input
         location = extract_zip(location)
-
-    mapping_file = validate_mapping(mapping, mapping_file)
-
-    errors, abouts = collect_inventory(location, mapping_file=mapping_file)
-
-    # Do not write the output if one of the ABOUT files has duplicated keys
-    # TODO: why do this check here?? Also if this is the place, we should list what the errors are.
-    dup_error_msg = u'Duplicated keys'
-    halt_output = False
-    for err in errors:
-        if dup_error_msg in err.message:
-            halt_output = True
-            break
-
-    if not halt_output:
-        write_errors = write_output(abouts=abouts, location=output, format=format)
-        for err in write_errors:
-            errors.append(err)
-    else:
-        if not quiet:
-            msg = u'Duplicated keys are not supported.\nPlease correct and re-run.'
-            click.echo(msg)
-
+    errors, abouts = collect_inventory(location)
+    write_errors = write_output(abouts=abouts, location=output, format=format)
+    errors.extend(write_errors)
     errors_count = report_errors(errors, quiet, verbose, log_file_loc=output + '-error.log')
     if not quiet:
         msg = 'Inventory collected in {output}.'.format(**locals())
@@ -272,19 +224,6 @@ def inventory(location, output, mapping, mapping_file,
     type=click.Path(exists=True, file_okay=False, readable=True, resolve_path=True),
     help='Path to a directory with reference license data and text files.')
 
-@click.option('--mapping',
-    is_flag=True,
-    help='Use the default built-in "mapping.config" file '
-         'with mapping between input keys and .ABOUT field names.'
-         'Cannot be combined with the --mapping-file option.')
-
-@click.option('--mapping-file',
-    metavar='FILE',
-    type=click.Path(exists=True, dir_okay=False, readable=True, resolve_path=True),
-    help='Path to an optional custom mapping FILE '
-         'with mapping between input keys and .ABOUT field names. '
-         'Cannot be combined with the --mapping option.')
-
 @click.option('-q', '--quiet',
     is_flag=True,
     help='Do not print error or warning messages.')
@@ -295,11 +234,7 @@ def inventory(location, output, mapping, mapping_file,
 
 @click.help_option('-h', '--help')
 
-def gen(location, output,
-        fetch_license,
-        reference,
-        mapping, mapping_file,
-        quiet, verbose):
+def gen(location, output, fetch_license, reference, quiet, verbose):
     """
 Generate .ABOUT files in OUTPUT from an inventory of .ABOUT files at LOCATION.
 
@@ -311,8 +246,6 @@ def gen(location, output,
         print_version()
         click.echo('Generating .ABOUT files...')
 
-    mapping_file = validate_mapping(mapping, mapping_file)
-
     if not location.endswith(('.csv', '.json',)):
         raise click.UsageError('ERROR: Invalid input file extension: must be one .csv or .json.')
 
@@ -321,7 +254,6 @@ def gen(location, output,
         base_dir=output,
         reference_dir=reference,
         fetch_license=fetch_license,
-        mapping_file=mapping_file
     )
 
     errors_count = report_errors(errors, quiet, verbose, log_file_loc=output + '-error.log')
@@ -378,25 +310,6 @@ def validate_template(ctx, param, value):
     metavar='<key>=<value>',
     help='Add variable text as key=value for use in a custom attribution template.')
 
-@click.option('--inventory',
-    metavar='FILE',
-    type=click.Path(exists=True, dir_okay=False, resolve_path=True),
-    help='Path to an optional JSON or CSV inventory FILE listing the '
-         'subset of .ABOUT files paths to consider when generating the attribution document.')
-
-@click.option('--mapping',
-    is_flag=True,
-    help='Use the default built-in "mapping.config" file '
-         'with mapping between input keys and .ABOUT field names.'
-         'Cannot be combined with the --mapping-file option.')
-
-@click.option('--mapping-file',
-    metavar='FILE',
-    type=click.Path(exists=True, dir_okay=False, readable=True, resolve_path=True),
-    help='Path to an optional custom mapping FILE '
-         'with mapping between input keys and .ABOUT field names. '
-         'Cannot be combined with the --mapping option.')
-
 @click.option('-q', '--quiet',
     is_flag=True,
     help='Do not print error or warning messages.')
@@ -407,9 +320,7 @@ def validate_template(ctx, param, value):
 
 @click.help_option('-h', '--help')
 
-def attrib(location, output, template, vartext,
-           inventory, mapping, mapping_file,
-           quiet, verbose):
+def attrib(location, output, template, vartext, quiet, verbose):
     """
 Generate an attribution document at OUTPUT using .ABOUT files at LOCATION.
 
@@ -421,20 +332,17 @@ def attrib(location, output, template, vartext,
         print_version()
         click.echo('Generating attribution...')
 
-    mapping_file = validate_mapping(mapping, mapping_file)
-
     # accept zipped ABOUT files as input
     if location.lower().endswith('.zip'):
         location = extract_zip(location)
 
-    errors, abouts = collect_inventory(location, mapping_file=mapping_file)
+    errors, abouts = collect_inventory(location)
 
     attrib_errors = generate_attribution_doc(
         abouts=abouts,
         output_location=output,
         template_loc=template,
         variables=vartext,
-        mapping_file=mapping_file,
     )
     errors.extend(attrib_errors)
 
@@ -531,7 +439,7 @@ def print_config_help(ctx, param, value):
 
 def transform(location, output, configuration, quiet, verbose):  # NOQA
     """
-Transform the CSV file at LOCATION by applying renamings, filters and checks 
+Transform the CSV file at LOCATION by applying renamings, filters and checks
 and write a new CSV to OUTPUT.
 
 LOCATION: Path to a CSV file.
@@ -609,23 +517,14 @@ def get_error_messages(errors, quiet=False, verbose=False):
                 messages .append(msg)
     return messages, severe_errors_count
 
-
-def filter_errors(errors, minimum_severity=WARNING):
-    """
-    Return a list of unique `errors` Error object filtering errors that have a
-    severity below `minimum_severity`.
-    """
-    return unique([e for e in errors if e.severity >= minimum_severity])
-
-
 ######################################################################
 # Misc
 ######################################################################
 
 def parse_key_values(key_values):
     """
     Given a list of "key=value" strings, return:
-    - a mapping {key: [value, value, ...]}
+    - a dict {key: [value, value, ...]}
     - a sorted list of unique error messages for invalid entries where there is
       a missing a key or value.
     """