Potential release of 7.2.1

 * stripped empty newline/spaces for about_resource
 * catch invalid license_expression

Signed-off-by: Chin Yeung Li <[email protected]>

Author: chinyeungli

CHANGELOG.rst

@@ -6,6 +6,8 @@ Changelog
     * Fix the syntax for setup's python_requires since newer version of
       setuptools (with newer packaging) versions do not accept star in the
       Python version spec.
+    * Stipped empty newline/spaces for about_resource fields
+    * Catch invalid license_expression
 
 
 2022-10-24

about.ABOUT

@@ -1,6 +1,6 @@
 about_resource: .
 name: AboutCode-toolkit
-version: 7.2.0
+version: 7.2.1
 author: Jillian Daguil, Chin Yeung Li, Philippe Ombredanne, Thomas Druez
 copyright: Copyright (c) nexB Inc.
 description: |

src/attributecode/__init__.py

@@ -20,7 +20,7 @@
 
 import saneyaml
 
-__version__ = '7.2.0'
+__version__ = '7.2.1'
 
 __about_spec_version__ = '3.2.3'
 

src/attributecode/attrib.py

@@ -39,6 +39,7 @@
 
 DEFAULT_LICENSE_SCORE = 100
 
+
 def generate(abouts, is_about_input, license_dict, scancode, min_license_score, template=None, vartext=None):
     """
     Generate an attribution text from an `abouts` list of About objects, a
@@ -55,7 +56,8 @@ def generate(abouts, is_about_input, license_dict, scancode, min_license_score,
         lineno, message = template_error
         error = Error(
             CRITICAL,
-            'Template validation error at line: {lineno}: "{message}"'.format(**locals())
+            'Template validation error at line: {lineno}: "{message}"'.format(
+                **locals())
         )
         errors.append(error)
         return error, None
@@ -87,10 +89,11 @@ def generate(abouts, is_about_input, license_dict, scancode, min_license_score,
                         filename = list(about.license_file.value.keys())[index]
                         text = list(about.license_file.value.values())[index]
                     else:
-                        error = Error(CRITICAL, 'No license file found for ' + name)
+                        error = Error(
+                            CRITICAL, 'No license file found for ' + name)
                         errors.append(error)
                         break
-                    if  about.license_url.value:
+                    if about.license_url.value:
                         url = about.license_url.value[index]
                     else:
                         url = ''
@@ -106,7 +109,6 @@ def generate(abouts, is_about_input, license_dict, scancode, min_license_score,
             license_object = License(key, name, filename, url, text)
             licenses_list.append(license_object)
 
-
     # We need special treatment for scancode input.
     # Each about_object may have duplicated license key and same/different license score
     # We will only keep the unique license key with the highest license score.
@@ -145,14 +147,18 @@ def generate(abouts, is_about_input, license_dict, scancode, min_license_score,
                             current_score = lic_score[index]
                             if current_score > previous_score:
                                 if matched_text_exist:
-                                    updated_dict[key] = (lic_score[index], lic_name[index], matched_text[index])
+                                    updated_dict[key] = (
+                                        lic_score[index], lic_name[index], matched_text[index])
                                 else:
-                                    updated_dict[key] = (lic_score[index], lic_name[index])
+                                    updated_dict[key] = (
+                                        lic_score[index], lic_name[index])
                         else:
                             if matched_text_exist:
-                                updated_dict[key] = (lic_score[index], lic_name[index], matched_text[index])
+                                updated_dict[key] = (
+                                    lic_score[index], lic_name[index], matched_text[index])
                             else:
-                                updated_dict[key] = (lic_score[index], lic_name[index])
+                                updated_dict[key] = (
+                                    lic_score[index], lic_name[index])
                         index = index + 1
                 updated_lic_key = []
                 updated_lic_name = []
@@ -200,7 +206,8 @@ def generate(abouts, is_about_input, license_dict, scancode, min_license_score,
             lic_name_expression = ' '.join(lic_name_expression_list)
 
             # Add the license name expression string into the about object as a custom field
-            custom_field = StringField(name='license_name_expression', value=lic_name_expression, present=True)
+            custom_field = StringField(
+                name='license_name_expression', value=lic_name_expression, present=True)
             setattr(about, 'license_name_expression', custom_field)
 
     # Sort the about objects by name
@@ -219,6 +226,7 @@ def generate(abouts, is_about_input, license_dict, scancode, min_license_score,
 
     return errors, rendered
 
+
 def get_license_file_key(license_text_name):
     if license_text_name.endswith('.LICENSE'):
         # See https://github.com/nexB/aboutcode-toolkit/issues/439
@@ -274,11 +282,16 @@ def generate_and_save(abouts, is_about_input, license_dict, output_location, sca
     for about in abouts:
         if not about.license_expression.value:
             continue
-        special_char_in_expression, lic_list = parse_license_expression(about.license_expression.value)
+        special_char_in_expression, lic_list, invalid_lic_exp = parse_license_expression(
+            about.license_expression.value)
         if special_char_in_expression:
             msg = (u"The following character(s) cannot be in the license_expression: " +
                    str(special_char_in_expression))
             errors.append(Error(ERROR, msg))
+        if invalid_lic_exp:
+            msg = (about.about_file_path + u": The following license_expression is invalid: " +
+                   invalid_lic_exp)
+            errors.append(Error(ERROR, msg))
     rendering_error, rendered = generate_from_file(
         abouts,
         is_about_input,

src/attributecode/gen.py

@@ -36,6 +36,7 @@
 from attributecode.util import UNC_PREFIX_POSIX
 from attributecode.util import unique
 from attributecode.util import load_scancode_json, load_csv, load_json, load_excel
+from attributecode.util import strip_inventory_value
 
 
 def check_duplicated_columns(location):
@@ -131,6 +132,7 @@ def load_inventory(location, from_attrib=False, base_dir=None, scancode=False, r
     """
     errors = []
     abouts = []
+    is_spreadsheet = False
 
     if base_dir:
         base_dir = util.to_posix(base_dir)
@@ -143,8 +145,10 @@ def load_inventory(location, from_attrib=False, base_dir=None, scancode=False, r
                 errors.extend(dup_cols_err)
                 return errors, abouts
             inventory = load_csv(location)
+            is_spreadsheet = True
         elif location.endswith('.xlsx'):
             dup_cols_err, inventory = load_excel(location)
+            is_spreadsheet = True
             if dup_cols_err:
                 errors.extend(dup_cols_err)
                 return errors, abouts
@@ -154,7 +158,14 @@ def load_inventory(location, from_attrib=False, base_dir=None, scancode=False, r
     try:
         arp_list = []
         errors = []
-        for component in inventory:
+
+        if is_spreadsheet:
+            # Only the .csv and .xlsx may have newline issue
+            stripped_inv = strip_inventory_value(inventory)
+        else:
+            stripped_inv = inventory
+
+        for component in stripped_inv:
             if not from_attrib:
                 arp = component['about_resource']
                 dup_err = check_duplicated_about_resource(arp, arp_list)
@@ -174,13 +185,15 @@ def load_inventory(location, from_attrib=False, base_dir=None, scancode=False, r
         if errors:
             return errors, abouts
     except Exception as e:
+        print("!!!!!!!!!!!!!!!!")
+        print(str(e))
         # TODO: why catch ALL Exception
         msg = "The essential field 'about_resource' is not found in the <input>"
         errors.append(Error(CRITICAL, msg))
         return errors, abouts
 
     custom_fields_list = []
-    for fields in inventory:
+    for fields in stripped_inv:
         # check does the input contains the required fields
         required_fields = model.About.required_fields