Enhance code for supporting license expression #275

 * Updated copyright date
 * Create a new function 'parse_license_expression'
 * Pass a sorted dictionary for rendering instead of having it to do in
the jinja2
 * The code did not copy 'notice_file' previously - Fixed.
 * Introduce a new 'license_expression' field. (I was using 'license'
field to put the license expression which may not be suitable)


Signed-off-by: Chin Yeung Li <[email protected]>

Author: chinyeungli

src/attributecode/attrib.py

@@ -2,7 +2,7 @@
 # -*- coding: utf8 -*-
 
 # ============================================================================
-#  Copyright (c) 2013-2016 nexB Inc. http://www.nexb.com/ - All rights reserved.
+#  Copyright (c) 2013-2017 nexB Inc. http://www.nexb.com/ - All rights reserved.
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
@@ -18,6 +18,7 @@
 from __future__ import print_function
 
 import codecs
+import collections
 import jinja2
 import os
 from posixpath import basename
@@ -29,6 +30,7 @@
 from attributecode import ERROR
 from attributecode import Error
 from attributecode.licenses import COMMON_LICENSES
+from attributecode.model import parse_license_expression
 from attributecode.util import add_unc
 
 
@@ -61,9 +63,10 @@ def generate(abouts, template_string=None):
                         else:
                             license_key = license_text_name
                         license_key_and_context[license_key] = about.license_file.value[license_text_name]
+                        sorted_license_key_and_context = collections.OrderedDict(sorted(license_key_and_context.items()))
                         license_text_name_and_key[license_text_name] = license_key
 
-        rendered = template.render(abouts=abouts, common_licenses=COMMON_LICENSES, license_key_and_context=license_key_and_context,
+        rendered = template.render(abouts=abouts, common_licenses=COMMON_LICENSES, license_key_and_context=sorted_license_key_and_context,
                                    license_text_name_and_key=license_text_name_and_key)
     except Exception, e:
         line = getattr(e, 'lineno', None)
@@ -167,6 +170,17 @@ def generate_and_save(abouts, output_location, mapping, template_loc=None,
                 if about.about_file_path == fp:
                     updated_abouts.append(about)
 
+    # Parse license_expression and save to the license list
+    for about in updated_abouts:
+        if about.license_expression.value:
+            special_char_in_expression, lic_list = parse_license_expression(about.license_expression.value)
+            if special_char_in_expression:
+                msg = (u"The following character(s) cannot be in the licesne_expression: " +
+                       str(special_char_in_expression))
+                errors.append(Error(ERROR, msg))
+            else:
+                about.license.value = lic_list
+
     rendered = generate_from_file(updated_abouts, template_loc=template_loc)
 
     if rendered:

src/attributecode/cmd.py

@@ -2,7 +2,7 @@
 # -*- coding: utf8 -*-
 
 # ============================================================================
-#  Copyright (c) 2013-2016 nexB Inc. http://www.nexb.com/ - All rights reserved.
+#  Copyright (c) 2013-2017 nexB Inc. http://www.nexb.com/ - All rights reserved.
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
@@ -136,12 +136,12 @@ def inventory(location, output, quiet, format):
                 '\nExample syntax:\n\n'
                 "about gen --fetch-license 'api_url' 'api_key'")
               )
-@click.option('--license-text-location', nargs=1,
+@click.option('--license-notice-text-location', nargs=1,
               type=click.Path(exists=True, dir_okay=True, readable=True, resolve_path=True),
               help="Copy the 'license_file' from the directory to the generated location")
 @click.option('--mapping', is_flag=True, help='Use for mapping between the input keys and the ABOUT field names - mapping.config')
 @click.option('-q', '--quiet', is_flag=True, help='Do not print any error/warning.')
-def gen(location, output, mapping, license_text_location, fetch_license, quiet):
+def gen(location, output, mapping, license_notice_text_location, fetch_license, quiet):
     """
 Given an inventory of ABOUT files at location, generate ABOUT files in base
 directory.
@@ -156,7 +156,7 @@ def gen(location, output, mapping, license_text_location, fetch_license, quiet):
         return
     click.echo('Generating ABOUT files...')
 
-    errors, abouts = attributecode.gen.generate(location, output, mapping, license_text_location, fetch_license)
+    errors, abouts = attributecode.gen.generate(location, output, mapping, license_notice_text_location, fetch_license)
 
     number_of_about_file = len(abouts)
     number_of_error = 0

src/attributecode/gen.py

@@ -97,7 +97,7 @@ def check_duplicated_about_file_path(inventory_dict):
     return errors
 
 
-def load_inventory(mapping, location, base_dir, license_text_location=None):
+def load_inventory(mapping, location, base_dir, license_notice_text_location=None):
     """
     Load the inventory file at location. Return a list of errors and a list of About
     objects validated against the base_dir.
@@ -150,7 +150,7 @@ def load_inventory(mapping, location, base_dir, license_text_location=None):
             loc = posixpath.join(base_dir, afp)
         about = model.About(about_file_path=afp)
         about.location = loc
-        ld_errors = about.load_dict(fields, base_dir, license_text_location, with_empty=False)
+        ld_errors = about.load_dict(fields, base_dir, license_notice_text_location, with_empty=False)
         # 'about_resource' field will be generated during the process.
         # No error need to be raise for the missing 'about_resource'.
         for e in ld_errors:
@@ -163,7 +163,7 @@ def load_inventory(mapping, location, base_dir, license_text_location=None):
     return errors, abouts
 
 
-def generate(location, base_dir, mapping, license_text_location, fetch_license, policy=None, conf_location=None,
+def generate(location, base_dir, mapping, license_notice_text_location, fetch_license, policy=None, conf_location=None,
              with_empty=False, with_absent=False):
     """
     Load ABOUT data from an inventory at csv_location. Write ABOUT files to
@@ -182,7 +182,7 @@ def generate(location, base_dir, mapping, license_text_location, fetch_license,
         gen_license = True
 
     bdir = to_posix(base_dir)
-    errors, abouts = load_inventory(mapping, location, bdir, license_text_location)
+    errors, abouts = load_inventory(mapping, location, bdir, license_notice_text_location)
 
     if gen_license:
         license_dict, err = model.pre_process_and_fetch_license_dict(abouts, api_url, api_key)

src/attributecode/model.py

@@ -46,7 +46,7 @@
 from attributecode import api
 from attributecode import saneyaml
 from attributecode import util
-from attributecode.util import add_unc, UNC_PREFIX, UNC_PREFIX_POSIX, on_windows, copy_license_files
+from attributecode.util import add_unc, UNC_PREFIX, UNC_PREFIX_POSIX, on_windows, copy_license_notice_files
 from license_expression import Licensing
 
 
@@ -363,7 +363,7 @@ def _validate(self, *args, **kwargs):
         """
         errors = super(PathField, self)._validate(*args, ** kwargs)
         self.base_dir = kwargs.get('base_dir')
-        self.license_text_location = kwargs.get('license_text_location')
+        self.license_notice_text_location = kwargs.get('license_notice_text_location')
 
         if self.base_dir:
             self.base_dir = util.to_posix(self.base_dir)
@@ -387,9 +387,9 @@ def _validate(self, *args, **kwargs):
             # the license files, if need to be copied, are located under the path
             # set from the 'license-text-location' option, so the tool should check
             # at the 'license-text-location' instead of the 'base_dir'
-            if self.base_dir or self.license_text_location:
-                if self.license_text_location:
-                    location = posixpath.join(self.license_text_location, path)
+            if self.base_dir or self.license_notice_text_location:
+                if self.license_notice_text_location:
+                    location = posixpath.join(self.license_notice_text_location, path)
                 else:
                     location = posixpath.join(self.base_dir, path)
                 location = util.to_native(location)
@@ -479,7 +479,7 @@ def _validate(self, *args, **kwargs):
             try:
                 # TODO: we have lots the location by replacing it with a text
                 location = add_unc(location)
-                text = codecs.open(location, encoding='utf-8').read()
+                text = codecs.open(location, encoding='utf-8', errors='ignore').read()
                 self.value[path] = text
             except Exception, e:
                 # only keep the first 100 char of the exception
@@ -584,14 +584,14 @@ def __eq__(self, other):
                 and self.value == other.value)
 
 
-def validate_fields(fields, base_dir, license_text_location=None):
+def validate_fields(fields, base_dir, license_notice_text_location=None):
     """
     Validate a sequence of Field objects. Return a list of errors.
     Validation may update the Field objects as needed as a side effect.
     """
     errors = []
     for f in fields:
-        val_err = f.validate(base_dir=base_dir, license_text_location=license_text_location)
+        val_err = f.validate(base_dir=base_dir, license_notice_text_location=license_notice_text_location)
         errors.extend(val_err)
     return errors
 
@@ -629,7 +629,8 @@ def create_fields(self):
             ('home_url', UrlField()),
             ('notes', StringField()),
 
-            ('license', StringField()),
+            ('license', ListField()),
+            ('license_expression', StringField()),
             ('license_name', StringField()),
             ('license_file', FileTextField()),
             ('license_url', UrlField()),
@@ -870,25 +871,25 @@ def hydrate(self, fields):
                         errors.append(Error(INFO, msg % locals()))
         return errors
 
-    def process(self, fields, base_dir=None, license_text_location=None):
+    def process(self, fields, base_dir=None, license_notice_text_location=None):
         """
         Hydrate and validate a sequence of field name/value tuples from an
         ABOUT file. Return a list of errors.
         """
         self.base_dir = base_dir
-        self.license_text_location = license_text_location
+        self.license_notice_text_location = license_notice_text_location
         afp = self.about_file_path
         errors = []
         hydratation_errors = self.hydrate(fields)
         errors.extend(hydratation_errors)
 
         # We want to copy the license_files before the validation
-        if license_text_location:
-            copy_license_files(fields, base_dir, license_text_location, afp)
+        if license_notice_text_location:
+            copy_license_notice_files(fields, base_dir, license_notice_text_location, afp)
 
         # we validate all fields, not only these hydrated
         all_fields = self.all_fields()
-        validation_errors = validate_fields(all_fields, self.base_dir, self.license_text_location)
+        validation_errors = validate_fields(all_fields, self.base_dir, self.license_notice_text_location)
         errors.extend(validation_errors)
 
         # do not forget to resolve about resource paths
@@ -937,7 +938,7 @@ def load_lines(self, lines, base_dir):
         self.errors = errors
         return errors
 
-    def load_dict(self, fields_dict, base_dir, license_text_location=None, with_empty=True):
+    def load_dict(self, fields_dict, base_dir, license_notice_text_location=None, with_empty=True):
         """
         Load the ABOUT file from a fields name/value mapping.
         If with_empty, create fields with no value for empty fields.
@@ -947,7 +948,7 @@ def load_dict(self, fields_dict, base_dir, license_text_location=None, with_empt
         fields = fields_dict.items()
         if not with_empty:
             fields = [(n, v) for n, v in fields_dict.items() if v]
-        errors = self.process(fields, base_dir, license_text_location)
+        errors = self.process(fields, base_dir, license_notice_text_location)
         self.errors = errors
         return errors
 
@@ -1009,8 +1010,9 @@ def dump_lic(self, location, license_dict):
         if not posixpath.exists(parent):
             os.makedirs(add_unc(parent))
 
-        if self.license.present and not self.license_file.present:
-            special_char_in_expression, lic_list = parse_license_expression(self.license.value)
+        if self.license_expression.present and not self.license_file.present:
+            special_char_in_expression, lic_list = parse_license_expression(self.license_expression.value)
+            self.license = lic_list
             if not special_char_in_expression:
                 for lic_key in lic_list:
                     try:
@@ -1347,8 +1349,8 @@ def pre_process_and_fetch_license_dict(abouts, api_url, api_key):
         auth_error = Error(ERROR, u"Authorization denied. Invalid '--api_key'. License generation is skipped.")
         if auth_error in errors:
             break
-        if about.license.present:
-            special_char_in_expression, lic_list = parse_license_expression(about.license.value)
+        if about.license_expression.present:
+            special_char_in_expression, lic_list = parse_license_expression(about.license_expression.value)
             if special_char_in_expression:
                 msg = (u"The following character(s) cannot be in the licesne_expression: " +
                        str(special_char_in_expression))

src/attributecode/util.py

@@ -411,13 +411,13 @@ def add_unc(location):
     return location
 
 
-def copy_license_files(fields, base_dir, license_text_location, afp):
+def copy_license_notice_files(fields, base_dir, license_notice_text_location, afp):
     lic_name = u''
     for key, value in fields:
-        if key == u'license_file':
+        if key == u'license_file' or key == u'notice_file':
             lic_name = value
 
-            from_lic_path = posixpath.join(to_posix(license_text_location), lic_name)
+            from_lic_path = posixpath.join(to_posix(license_notice_text_location), lic_name)
             about_file_dir = dirname(to_posix(afp)).lstrip('/')
             to_lic_path = posixpath.join(to_posix(base_dir), about_file_dir)