Merge branch 'load_json_issue' into develop

Author: chinyeungli

src/attributecode/attrib.py

@@ -143,18 +143,20 @@ def generate_from_file(abouts, template_loc=None):
     return generate(abouts, template_string=tpls)
 
 
-def generate_and_save(abouts, output_location, use_mapping=False,
+def generate_and_save(abouts, output_location, use_mapping=False, mapping_file=None,
                       template_loc=None, inventory_location=None):
     """
     Generate attribution file using the `abouts` list of About object
     at `output_location`.
 
+    Optionally use the mapping.config file if `use_mapping` is True.
+
+    Optionally use the custom mapping file if mapping_file is set.
+
     Use the optional `template_loc` custom temaplte or a default template.
 
     Optionally filter `abouts` object based on the inventory JSON or
     CSV at `inventory_location`.
-
-    Optionally use the mapping.config file is `use_mapping` is True
     """
     updated_abouts = []
     lstrip_afp = []
@@ -178,7 +180,7 @@ def generate_and_save(abouts, output_location, use_mapping=False,
             try:
                 # Return a list which contains only the about file path
                 about_list = attributecode.util.get_about_file_path(
-                    inventory_location, use_mapping=use_mapping)
+                    inventory_location, use_mapping=use_mapping, mapping_file=mapping_file)
             # FIXME: why catching all exceptions?
             except Exception:
                 # 'about_file_path' key/column doesn't exist

src/attributecode/cmd.py

@@ -115,6 +115,10 @@ def cli():
 @click.option('--mapping', is_flag=True,
     help='Use file mapping.config to collect the defined not supported fields in ABOUT files.')
 
+@click.option('--mapping-file', metavar='FILE', nargs=1,
+    type=click.Path(exists=True, dir_okay=True, readable=True, resolve_path=True),
+    help='Use a custom mapping file with mapping between input keys and ABOUT field names.')
+
 @click.option('--show-all', is_flag=True, default=False,
     help='Show all errors and warnings. '
         'By default, the tool only prints these '
@@ -128,7 +132,7 @@ def cli():
 
 @click.help_option('-h', '--help')
 
-def inventory(location, output, mapping, quiet, format, show_all):
+def inventory(location, output, mapping, mapping_file, quiet, format, show_all):
     """
 Collect a JSON or CSV inventory of components from .ABOUT files.
 
@@ -151,7 +155,7 @@ def inventory(location, output, mapping, quiet, format, show_all):
         # accept zipped ABOUT files as input
         location = extract_zip(location)
 
-    errors, abouts = model.collect_inventory(location, use_mapping=mapping)
+    errors, abouts = model.collect_inventory(location, use_mapping=mapping, mapping_file=mapping_file)
 
     write_errors = model.write_output(abouts, output, format)
     for err in write_errors:
@@ -176,7 +180,7 @@ def inventory(location, output, mapping, quiet, format, show_all):
 @click.argument('output', nargs=1, required=True,
     type=click.Path(exists=True, writable=True, dir_okay=True, resolve_path=True))
 
-@click.option('--fetch-license', type=str, nargs=2, metavar='<key>',
+@click.option('--fetch-license', type=str, nargs=2, metavar='KEY',
     help=('Fetch licenses text from a DejaCode API. and create <license>.LICENSE side-by-side '
         'with the generated .ABOUT file using data fetched from a DejaCode License Library. '
         'The "license" key is needed in the input. '
@@ -193,7 +197,11 @@ def inventory(location, output, mapping, quiet, format, show_all):
     help="Copy the 'license_file' from the directory to the generated location.")
 
 @click.option('--mapping', is_flag=True,
-    help='Use file mapping.config with mapping between input keys and ABOUT field names.')
+    help='Use the default file mapping.config with mapping between input keys and ABOUT field names.')
+
+@click.option('--mapping-file', metavar='FILE', nargs=1,
+    type=click.Path(exists=True, dir_okay=True, readable=True, resolve_path=True),
+    help='Use a custom mapping file with mapping between input keys and ABOUT field names.')
 
 @click.option('--show-all', is_flag=True, default=False,
     help='Show all errors and warnings. '
@@ -208,7 +216,7 @@ def inventory(location, output, mapping, quiet, format, show_all):
 
 @click.help_option('-h', '--help')
 
-def gen(location, output, mapping, license_notice_text_location, fetch_license,
+def gen(location, output, mapping, mapping_file, license_notice_text_location, fetch_license,
         quiet, show_all):
     """
 Generate .ABOUT files in OUTPUT directory from a JSON or CSV inventory of .ABOUT files at LOCATION.
@@ -227,7 +235,7 @@ def gen(location, output, mapping, license_notice_text_location, fetch_license,
 
     errors, abouts = gen_generate(
         location=location, base_dir=output, license_notice_text_location=license_notice_text_location,
-        fetch_license=fetch_license, use_mapping=mapping)
+        fetch_license=fetch_license, use_mapping=mapping, mapping_file=mapping_file)
 
     about_count = len(abouts)
     error_count = 0
@@ -266,6 +274,10 @@ def gen(location, output, mapping, license_notice_text_location, fetch_license,
     help='Use the file "mapping.config" with mappings between the CSV '
         'inventory columns names and .ABOUT field names.')
 
+@click.option('--mapping-file', metavar='FILE', nargs=1,
+    type=click.Path(exists=True, dir_okay=True, readable=True, resolve_path=True),
+    help='Use a custom mapping file with mapping between input keys and ABOUT field names.')
+
 @click.option('--show-all', is_flag=True, default=False,
     help='Show all errors and warnings. '
         'By default, the tool only prints these '
@@ -282,7 +294,7 @@ def gen(location, output, mapping, license_notice_text_location, fetch_license,
 
 @click.help_option('-h', '--help')
 
-def attrib(location, output, template, mapping, inventory, quiet, show_all):
+def attrib(location, output, template, mapping, mapping_file, inventory, quiet, show_all):
     """
 Generate an attribution document at OUTPUT using .ABOUT files at LOCATION.
 
@@ -297,10 +309,10 @@ def attrib(location, output, template, mapping, inventory, quiet, show_all):
     if location.lower().endswith('.zip'):
         location = extract_zip(location)
 
-    inv_errors, abouts = model.collect_inventory(location, use_mapping=mapping)
+    inv_errors, abouts = model.collect_inventory(location, use_mapping=mapping, mapping_file=mapping_file)
     no_match_errors = attrib_generate_and_save(
         abouts=abouts, output_location=output,
-        use_mapping=mapping, template_loc=template,
+        use_mapping=mapping, mapping_file=mapping_file, template_loc=template,
         inventory_location=inventory)
 
     if not no_match_errors:

src/attributecode/gen.py

@@ -107,14 +107,15 @@ def check_duplicated_about_file_path(inventory_dict):
 
 
 def load_inventory(location, base_dir, license_notice_text_location=None,
-                   use_mapping=False):
+                   use_mapping=False, mapping_file=None):
     """
     Load the inventory file at `location` for ABOUT and LICENSE files
     stored in the `base_dir`. Return a list of errors and a list of
     About objects validated against the base_dir.
     Optionally use `license_notice_text_location` as the location of
     license and notice texts.
-    Optionally use mappings for field names if `use_mapping` is True.
+    Optionally use mappings for field names if `use_mapping` is True
+    or a custom mapping_file if provided.
     """
     errors = []
     abouts = []
@@ -124,9 +125,9 @@ def load_inventory(location, base_dir, license_notice_text_location=None,
         if dup_cols_err:
             errors.extend(dup_cols_err)
             return errors, abouts
-        inventory = util.load_csv(location, use_mapping)
+        inventory = util.load_csv(location, use_mapping, mapping_file)
     else:
-        inventory = util.load_json(location, use_mapping)
+        inventory = util.load_json(location, use_mapping, mapping_file)
 
     try:
         dup_about_paths_err = check_duplicated_about_file_path(inventory)
@@ -165,7 +166,9 @@ def load_inventory(location, base_dir, license_notice_text_location=None,
         about = model.About(about_file_path=afp)
         about.location = loc
         running_inventory = False
-        ld_errors = about.load_dict(fields, base_dir, running_inventory, use_mapping, license_notice_text_location, with_empty=False)
+        ld_errors = about.load_dict(fields, base_dir, running_inventory,
+                                    use_mapping, mapping_file, license_notice_text_location,
+                                    with_empty=False)
         # 'about_resource' field will be generated during the process.
         # No error need to be raise for the missing 'about_resource'.
         for e in ld_errors:
@@ -180,7 +183,7 @@ def load_inventory(location, base_dir, license_notice_text_location=None,
 
 def generate(location, base_dir, license_notice_text_location=None,
              fetch_license=False, policy=None, conf_location=None,
-             with_empty=False, with_absent=False, use_mapping=False):
+             with_empty=False, with_absent=False, use_mapping=False, mapping_file=None):
     """
     Load ABOUT data from a CSV inventory at `location`. Write ABOUT files to
     base_dir using policy flags and configuration file at conf_location.
@@ -203,7 +206,8 @@ def generate(location, base_dir, license_notice_text_location=None,
         location=location,
         base_dir=bdir,
         license_notice_text_location=license_notice_text_location,
-        use_mapping=use_mapping)
+        use_mapping=use_mapping,
+        mapping_file=mapping_file)
 
     if gen_license:
         license_dict, err = model.pre_process_and_fetch_license_dict(abouts, api_url, api_key)

src/attributecode/model.py

@@ -710,7 +710,7 @@ def create_fields(self):
             field.name = name
             setattr(self, name, field)
 
-    def __init__(self, location=None, about_file_path=None, use_mapping=False):
+    def __init__(self, location=None, about_file_path=None, use_mapping=False, mapping_file=None):
         self.create_fields()
         self.custom_fields = OrderedDict()
 
@@ -724,7 +724,7 @@ def __init__(self, location=None, about_file_path=None, use_mapping=False):
         self.base_dir = None
         if self.location:
             self.base_dir = os.path.dirname(location)
-            self.load(location, use_mapping)
+            self.load(location, use_mapping, mapping_file)
 
     def __repr__(self):
         return repr(self.all_fields())
@@ -842,7 +842,7 @@ def as_dict(self, with_paths=False, with_absent=True, with_empty=True):
             as_dict[field.name] = field.serialized_value()
         return as_dict
 
-    def hydrate(self, fields, use_mapping=False):
+    def hydrate(self, fields, use_mapping=False, mapping_file=None):
         """
         Process an iterable of field (name, value) tuples. Update or create
         Fields attributes and the fields and custom fields dictionaries.
@@ -852,9 +852,8 @@ def hydrate(self, fields, use_mapping=False):
         seen_fields = OrderedDict()
 
         mapping = {}
-        if use_mapping:
-            mapping = util.get_mapping()
-
+        if use_mapping or mapping_file:
+            mapping = util.get_mapping(mapping_file)
         for name, value in fields:
             orig_name = name
             name = name.lower()
@@ -879,7 +878,7 @@ def hydrate(self, fields, use_mapping=False):
                 standard_field.present = True
                 continue
 
-            if not use_mapping:
+            if not use_mapping and not mapping_file:
                 if not name == self.about_file_path_attr:
                     msg = (u'Field %(orig_name)s is not a supported field and is ignored.')
                     errors.append(Error(INFO, msg % locals()))
@@ -935,7 +934,8 @@ def hydrate(self, fields, use_mapping=False):
         return errors
 
     def process(self, fields, about_file_path, running_inventory=False,
-                base_dir=None, license_notice_text_location=None, use_mapping=False):
+                base_dir=None, license_notice_text_location=None,
+                use_mapping=False, mapping_file=None):
         """
         Hydrate and validate a sequence of field name/value tuples from an
         ABOUT file. Return a list of errors.
@@ -944,7 +944,7 @@ def process(self, fields, about_file_path, running_inventory=False,
         self.license_notice_text_location = license_notice_text_location
         afp = self.about_file_path
         errors = []
-        hydratation_errors = self.hydrate(fields, use_mapping=use_mapping)
+        hydratation_errors = self.hydrate(fields, use_mapping=use_mapping, mapping_file=mapping_file)
         errors.extend(hydratation_errors)
 
         # We want to copy the license_files before the validation
@@ -965,7 +965,7 @@ def process(self, fields, about_file_path, running_inventory=False,
         # self.about_resource.resolve(self.about_file_path)
         return errors
 
-    def load(self, location, use_mapping=False):
+    def load(self, location, use_mapping=False, mapping_file=None):
         """
         Read, parse and process the ABOUT file at location.
         Return a list of errors and update self with errors.
@@ -988,7 +988,7 @@ def load(self, location, use_mapping=False):
             and then join with the 'about_resource_path'
             """
             running_inventory = True
-            errs = self.load_dict(saneyaml.load(input_text), base_dir, running_inventory, use_mapping)
+            errs = self.load_dict(saneyaml.load(input_text), base_dir, running_inventory, use_mapping, mapping_file)
             errors.extend(errs)
         except Exception as e:
             msg = 'Cannot load invalid ABOUT file: %(location)r: %(e)r\n' + str(e)
@@ -1018,7 +1018,8 @@ def load_lines(self, lines, base_dir):
         self.errors = errors
         return errors
 
-    def load_dict(self, fields_dict, base_dir, running_inventory=False, use_mapping=False,
+    def load_dict(self, fields_dict, base_dir, running_inventory=False,
+                  use_mapping=False, mapping_file=None,
                   license_notice_text_location=None, with_empty=True):
         """
         Load the ABOUT file from a fields name/value mapping.
@@ -1032,7 +1033,7 @@ def load_dict(self, fields_dict, base_dir, running_inventory=False, use_mapping=
             fields = [(n, v) for n, v in fields_dict.items() if v]
         errors = self.process(
             fields, about_file_path, running_inventory, base_dir, 
-            license_notice_text_location, use_mapping)
+            license_notice_text_location, use_mapping, mapping_file)
         self.errors = errors
         return errors
 
@@ -1190,7 +1191,7 @@ def parse(lines):
     return errors, fields
 
 
-def collect_inventory(location, use_mapping=False):
+def collect_inventory(location, use_mapping=False, mapping_file=None):
     """
     Collect ABOUT files at location and return a list of errors and a list of
     About objects.
@@ -1205,7 +1206,7 @@ def collect_inventory(location, use_mapping=False):
     abouts = []
     for about_loc in about_locations:
         about_file_path = util.get_relative_path(input_location, about_loc)
-        about = About(about_loc, about_file_path, use_mapping)
+        about = About(about_loc, about_file_path, use_mapping, mapping_file)
         # Insert about_file_path reference to the error
         for severity, message in about.errors:
             msg = (about_file_path + ": " + message)