Merge with #479

Signed-off-by: Chin Yeung Li <[email protected]>

Author: chinyeungli

setup.cfg

@@ -51,6 +51,7 @@ install_requires =
     license_expression >= 0.94
     openpyxl
     packageurl_python >= 0.9.0
+    openpyxl
 
 setup_requires = setuptools_scm[toml] >= 4
 python_requires = >=3.6.*, <4

src/attributecode/attrib.py

@@ -26,16 +26,20 @@
 from attributecode import ERROR
 from attributecode import Error
 from attributecode.licenses import COMMON_LICENSES
-from attributecode.model import detect_special_char
 from attributecode.model import parse_license_expression
+from attributecode.model import License
 from attributecode.util import add_unc
 from attributecode.attrib_util import multi_sort
 
 DEFAULT_TEMPLATE_FILE = os.path.join(
     os.path.dirname(os.path.realpath(__file__)), '../../templates', 'default_html.template')
 
+DEFAULT_TEMPLATE_SCANCODE_FILE = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)), '../../templates', 'scancode_html.template')
 
-def generate(abouts, template=None, variables=None):
+DEFAULT_LICENSE_SCORE = 100
+
+def generate(abouts, is_about_input, license_dict, scancode, min_license_score, template=None, variables=None):
     """
     Generate an attribution text from an `abouts` list of About objects, a
     `template` template text and a `variables` optional dict of extra
@@ -56,121 +60,122 @@ def generate(abouts, template=None, variables=None):
         return error, None
 
     template = jinja2.Template(template)
+    # Get the current UTC time
+    utcnow = datetime.datetime.utcnow()
 
-    try:
-        captured_license = []
-        license_file_key_and_context = {}
-        sorted_license_file_key_and_context = {}
-        license_file_name_and_license_file_key = {}
-        license_key_and_license_name = {}
-        license_name_and_license_key = {}
-        license_key_and_license_file_name = {}
-        license_file_key_and_license_key = {}
-        # FIXME: This need to be simplified
+    licenses_list = []
+    lic_name_expression_list = []
+    if is_about_input:
         for about in abouts:
             # about.license_file.value is a OrderDict with license_file_name as
             # the key and the license text as the value
-            if about.license_file:
-                # We want to create a dictionary which have the license file key as
-                # the key and license text as the value
-                # The reason we want to use license file key as the key instead of the
-                # license key is because there is a scenario such that the input only provide
-                # license_file but not license_key
-                # The license file key is bascially a license_key or a license file
-                # name if it's not generated from DJE. The reason for not using
-                # license file name as the key at the first place is because
-                # we need the license_key to match with the common license list
-                for license_file_name in about.license_file.value:
-                    if not license_file_name in captured_license:
-                        captured_license.append(license_file_name)
-                        license_file_key = get_license_file_key(license_file_name)
-                        license_file_key_and_context[license_file_key] = about.license_file.value[license_file_name]
-                        sorted_license_file_key_and_context = collections.OrderedDict(sorted(license_file_key_and_context.items()))
-                        license_file_name_and_license_file_key[license_file_name] = license_file_key
-
-            lic_list = []
-            lic_name_list = []
-            lic_name_expression_list = []
-            # Convert/map the key to name
-            if about.license_name.value:
-                if about.license_expression.value or about.license_key.value:
-                    if about.license_expression.value:
-                        special_char, lic_list = parse_license_expression(about.license_expression.value)
-                        about.license_key.value = lic_list
-                    else:
-                        lic_list = about.license_key.value
-                        special_char = []
-                        for lic in lic_list:
-                            special_char_list = detect_special_char(lic)
-                            if special_char_list:
-                                for char in special_char_list:
-                                    special_char.append(char)
-                    if special_char:
-                        error = Error(CRITICAL, 'Special character(s) are not allowed in '
-                                      'license_expression or license_key: %s' % special_char)
-                        return error, ''
+            index = 0
+            for lic_name in about.license_name.value:
+                if about.license_key.value:
+                    key = about.license_key.value[index]
                 else:
-                    # No license_key or license_expression present. We will put
-                    # None as the value of license key
-                    about.license_key.value = about.license_file.value.keys()
-                    lic_list = about.license_file.value.keys()
-
-                lic_name_list = about.license_name.value
-
-                # The order of the license_name and key should be the same
-                # The length for both list should be the same
-                assert len(lic_name_list) == len(lic_list)
-
-                # Map the license key to license name
-                index_for_license_name_list = 0
-                for key in lic_list:
-                    license_key_and_license_file_name[key] = list(about.license_file.value.keys())[index_for_license_name_list]
-                    license_key_and_license_name[key] = lic_name_list[index_for_license_name_list]
-                    license_name_and_license_key[lic_name_list[index_for_license_name_list]] = key
-                    license_file_key = license_file_name_and_license_file_key[license_key_and_license_file_name[key]]
-                    license_file_key_and_license_key[license_file_key] = key
-                    index_for_license_name_list = index_for_license_name_list + 1
-
-                # Create a license expression with license name instead of key
-                for segment in about.license_expression.value.split():
-                    if segment in license_key_and_license_name:
-                        lic_name_expression_list.append(license_key_and_license_name[segment])
+                    key = lic_name
+                captured = False
+                for lic in licenses_list:
+                    if key in lic.key:
+                        captured = True
+                if not captured or not licenses_list:
+                    name = lic_name
+                    filename = list(about.license_file.value.keys())[index]
+                    if  about.license_url.value:
+                        url = about.license_url.value[index]
                     else:
-                        lic_name_expression_list.append(segment)
-
-                # Join the license name expression into a single string
-                lic_name_expression = ' '.join(lic_name_expression_list)
-
-                # Add the license name expression string into the about object
-                about.license_name_expression = lic_name_expression
-
-        # Get the current UTC time
-        utcnow = datetime.datetime.utcnow()
-        rendered = template.render(
-            abouts=abouts, common_licenses=COMMON_LICENSES,
-            license_file_key_and_context=sorted_license_file_key_and_context,
-            license_file_key_and_license_key=license_file_key_and_license_key,
-            license_file_name_and_license_file_key=license_file_name_and_license_file_key,
-            license_key_and_license_file_name=license_key_and_license_file_name,
-            license_key_and_license_name=license_key_and_license_name,
-            license_name_and_license_key=license_name_and_license_key,
-            utcnow=utcnow,
-            tkversion=__version__,
-            variables=variables
-        )
-    except Exception as e:
-        lineno = getattr(e, 'lineno', '') or ''
-        if lineno:
-            lineno = ' at line: {}'.format(lineno)
-        err = getattr(e, 'message', '') or ''
-#        error = Error(
-#            CRITICAL,
-#            'Template processing error {lineno}: {err}'.format(**locals()),
-#        )
-        error = Error(
-            CRITICAL,
-            'Template processing error:' + str(e),
-        )
+                        url = ''
+                    text = list(about.license_file.value.values())[index]
+                    license_object = License(key, name, filename, url, text)
+                    licenses_list.append(license_object) 
+                index = index + 1
+    else:
+        for key in license_dict:
+            name = license_dict[key][0]
+            filename = license_dict[key][1]
+            text = license_dict[key][2]
+            url = license_dict[key][3]
+            license_object = License(key, name, filename, url, text)
+            licenses_list.append(license_object)
+
+
+    # We need special treatment for scancode input.
+    # Each about_object may have duplicated license key and same/different license score
+    # We will only keep the unique license key with the highest license score.
+    # The process will update the license_key, license_name and license_score.
+    if scancode:
+        meet_score_licenses_list = []
+        for about in abouts:
+            # We will use a dictionary to keep the unique license key
+            # which the dictionary key is the license key and the dictionary value
+            # is (lic_score, lic_name)
+            if about.license_key.value:
+                updated_dict = {}
+                lic_key = about.license_key.value
+                lic_name = about.license_name.value
+                lic_score = about.license_score.value
+                assert len(lic_key) == len(lic_name)
+                assert len(lic_key) == len(lic_score)
+                if lic_key:
+                    index = 0
+                    for key in lic_key:
+                        if key in updated_dict:
+                            previous_score, _name = updated_dict[key]
+                            current_score = lic_score[index]
+                            if current_score > previous_score:
+                                updated_dict[key] = (lic_score[index], lic_name[index])
+                        else:
+                            updated_dict[key] = (lic_score[index], lic_name[index])
+                        index = index + 1
+                updated_lic_key = []
+                updated_lic_name = []
+                updated_lic_score = []
+                for lic in updated_dict:
+                    score, name = updated_dict[lic]
+                    if score >= min_license_score:
+                        updated_lic_key.append(lic)
+                        updated_lic_score.append(score)
+                        updated_lic_name.append(name)
+                        if not lic in meet_score_licenses_list:
+                            meet_score_licenses_list.append(lic)
+                about.license_key.value = updated_lic_key
+                about.license_name.value = updated_lic_name
+                about.license_score.value = updated_lic_score
+
+        for lic in licenses_list:
+            if not lic.key in meet_score_licenses_list:
+                licenses_list.remove(lic)
+
+    for about in abouts:
+        # Create a license expression with license name
+        if about.license_expression.value:
+            for segment in about.license_expression.value.split():
+                not_lic = True
+                for lic in licenses_list:
+                    if segment == lic.key:
+                        lic_name_expression_list.append(lic.name)
+                        not_lic = False
+                        break
+                if not_lic:
+                    lic_name_expression_list.append(segment)
+            # Join the license name expression into a single string
+            lic_name_expression = ' '.join(lic_name_expression_list)
+        
+            # Add the license name expression string into the about object as a list
+            about.license_name_expression = lic_name_expression
+
+    # Sort the license object by key
+    licenses_list = sorted(licenses_list, key=lambda x: x.key)
+
+    rendered = template.render(
+        abouts=abouts, 
+        common_licenses=COMMON_LICENSES,
+        licenses_list=licenses_list,
+        utcnow=utcnow,
+        tkversion=__version__,
+        variables=variables
+    )
     return error, rendered
 
 
@@ -195,7 +200,7 @@ def check_template(template_string):
         return e.lineno, e.message
 
 
-def generate_from_file(abouts, template_loc=DEFAULT_TEMPLATE_FILE, variables=None):
+def generate_from_file(abouts, is_about_input, license_dict, scancode, min_license_score, template_loc=None, variables=None):
     """
     Generate an attribution text from an `abouts` list of About objects, a
     `template_loc` template file location and a `variables` optional
@@ -204,14 +209,19 @@ def generate_from_file(abouts, template_loc=DEFAULT_TEMPLATE_FILE, variables=Non
     Return a tuple of (error, attribution text) where error is an Error object
     or None and attribution text is the generated text or None.
     """
-
-    template_loc = add_unc(template_loc)
+    if not template_loc:
+        if scancode:
+            template_loc = add_unc(DEFAULT_TEMPLATE_SCANCODE_FILE)
+        else:
+            template_loc = add_unc(DEFAULT_TEMPLATE_FILE)
+    else:
+        template_loc = add_unc(template_loc)
     with io.open(template_loc, encoding='utf-8') as tplf:
         tpls = tplf.read()
-    return generate(abouts, template=tpls, variables=variables)
+    return generate(abouts, is_about_input, license_dict, scancode, min_license_score, template=tpls, variables=variables)
 
 
-def generate_and_save(abouts, output_location, template_loc=None, variables=None):
+def generate_and_save(abouts, is_about_input, license_dict, output_location, scancode=False, min_license_score=0, template_loc=None, variables=None):
     """
     Generate an attribution text from an `abouts` list of About objects, a
     `template_loc` template file location and a `variables` optional
@@ -220,7 +230,6 @@ def generate_and_save(abouts, output_location, template_loc=None, variables=None
     Return a list of Error objects if any.
     """
     errors = []
-
     # Parse license_expression and save to the license list
     for about in abouts:
         if not about.license_expression.value:
@@ -230,11 +239,14 @@ def generate_and_save(abouts, output_location, template_loc=None, variables=None
             msg = (u"The following character(s) cannot be in the license_expression: " +
                    str(special_char_in_expression))
             errors.append(Error(ERROR, msg))
-
     rendering_error, rendered = generate_from_file(
         abouts,
+        is_about_input,
+        license_dict,
+        scancode=scancode,
+        min_license_score=min_license_score,
         template_loc=template_loc,
-        variables=variables
+        variables=variables,
     )
 
     if rendering_error:

src/attributecode/attrib_util.py

@@ -15,7 +15,7 @@
 # ============================================================================
 
 from jinja2 import Environment
-from jinja2.filters import environmentfilter
+from jinja2.filters import pass_environment
 from jinja2.filters import make_attrgetter
 from jinja2.filters import ignore_case
 from jinja2.filters import FilterArgumentError
@@ -38,7 +38,7 @@ def get_template(template_text):
     return env.from_string(template_text)
 
 
-@environmentfilter
+@pass_environment
 def multi_sort(environment, value, reverse=False, case_sensitive=False,
                attributes=None):
     """
@@ -72,7 +72,7 @@ def key(v):
     return sorted(value, key=key, reverse=reverse)
 
 
-@environmentfilter
+@pass_environment
 def unique_together(environment, value, case_sensitive=False, attributes=None):
     """
     Return a list of unique items from an iterable. Unicity is checked when