#479 - Code related to use scancode scan as the input

 * Create a scancode default template
 * Use different default template for usual and scancode input

I was trying to use the template to handle min_license_score situation,
but it's too hard and too complex. Therefore, I updated the code to
handle that.

Tests are expected to fail as I haven't updated the tests yet.

Signed-off-by: Chin Yeung Li <[email protected]>

Author: chinyeungli

src/attributecode/attrib.py

@@ -34,9 +34,12 @@
 DEFAULT_TEMPLATE_FILE = os.path.join(
     os.path.dirname(os.path.realpath(__file__)), '../../templates', 'default_html.template')
 
+DEFAULT_TEMPLATE_SCANCODE_FILE = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)), '../../templates', 'scancode_html.template')
+
 DEFAULT_LICENSE_SCORE = 100
 
-def generate(abouts, is_about_input, license_dict, min_license_score, template=None, variables=None):
+def generate(abouts, is_about_input, license_dict, scancode, min_license_score, template=None, variables=None):
     """
     Generate an attribution text from an `abouts` list of About objects, a
     `template` template text and a `variables` optional dict of extra
@@ -96,6 +99,54 @@ def generate(abouts, is_about_input, license_dict, min_license_score, template=N
             license_object = License(key, name, filename, url, text)
             licenses_list.append(license_object)
 
+
+    # We need special treatment for scancode input.
+    # Each about_object may have duplicated license key and same/different license score
+    # We will only keep the unique license key with the highest license score.
+    # The process will update the license_key, license_name and license_score.
+    if scancode:
+        meet_score_licenses_list = []
+        for about in abouts:
+            # We will use a dictionary to keep the unique license key
+            # which the dictionary key is the license key and the dictionary value
+            # is (lic_score, lic_name)
+            if about.license_key.value:
+                updated_dict = {}
+                lic_key = about.license_key.value
+                lic_name = about.license_name.value
+                lic_score = about.license_score.value
+                assert len(lic_key) == len(lic_name)
+                assert len(lic_key) == len(lic_score)
+                if lic_key:
+                    index = 0
+                    for key in lic_key:
+                        if key in updated_dict:
+                            previous_score, _name = updated_dict[key]
+                            current_score = lic_score[index]
+                            if current_score > previous_score:
+                                updated_dict[key] = (lic_score[index], lic_name[index])
+                        else:
+                            updated_dict[key] = (lic_score[index], lic_name[index])
+                        index = index + 1
+                updated_lic_key = []
+                updated_lic_name = []
+                updated_lic_score = []
+                for lic in updated_dict:
+                    score, name = updated_dict[lic]
+                    if score >= min_license_score:
+                        updated_lic_key.append(lic)
+                        updated_lic_score.append(score)
+                        updated_lic_name.append(name)
+                        if not lic in meet_score_licenses_list:
+                            meet_score_licenses_list.append(lic)
+                about.license_key.value = updated_lic_key
+                about.license_name.value = updated_lic_name
+                about.license_score.value = updated_lic_score
+
+        for lic in licenses_list:
+            if not lic.key in meet_score_licenses_list:
+                licenses_list.remove(lic)
+
     for about in abouts:
         # Create a license expression with license name
         if about.license_expression.value:
@@ -121,7 +172,6 @@ def generate(abouts, is_about_input, license_dict, min_license_score, template=N
         abouts=abouts, 
         common_licenses=COMMON_LICENSES,
         licenses_list=licenses_list,
-        min_license_score=min_license_score,
         utcnow=utcnow,
         tkversion=__version__,
         variables=variables
@@ -150,7 +200,7 @@ def check_template(template_string):
         return e.lineno, e.message
 
 
-def generate_from_file(abouts, is_about_input, license_dict, min_license_score, template_loc=DEFAULT_TEMPLATE_FILE, variables=None):
+def generate_from_file(abouts, is_about_input, license_dict, scancode, min_license_score, template_loc=None, variables=None):
     """
     Generate an attribution text from an `abouts` list of About objects, a
     `template_loc` template file location and a `variables` optional
@@ -159,13 +209,19 @@ def generate_from_file(abouts, is_about_input, license_dict, min_license_score,
     Return a tuple of (error, attribution text) where error is an Error object
     or None and attribution text is the generated text or None.
     """
-    template_loc = add_unc(template_loc)
+    if not template_loc:
+        if scancode:
+            template_loc = add_unc(DEFAULT_TEMPLATE_SCANCODE_FILE)
+        else:
+            template_loc = add_unc(DEFAULT_TEMPLATE_FILE)
+    else:
+        template_loc = add_unc(template_loc)
     with io.open(template_loc, encoding='utf-8') as tplf:
         tpls = tplf.read()
-    return generate(abouts, is_about_input, license_dict, min_license_score, template=tpls, variables=variables)
+    return generate(abouts, is_about_input, license_dict, scancode, min_license_score, template=tpls, variables=variables)
 
 
-def generate_and_save(abouts, is_about_input, license_dict, output_location, min_license_score=0, template_loc=None, variables=None):
+def generate_and_save(abouts, is_about_input, license_dict, output_location, scancode=False, min_license_score=0, template_loc=None, variables=None):
     """
     Generate an attribution text from an `abouts` list of About objects, a
     `template_loc` template file location and a `variables` optional
@@ -187,6 +243,7 @@ def generate_and_save(abouts, is_about_input, license_dict, output_location, min
         abouts,
         is_about_input,
         license_dict,
+        scancode=scancode,
         min_license_score=min_license_score,
         template_loc=template_loc,
         variables=variables,

src/attributecode/cmd.py

@@ -279,7 +279,7 @@ def gen(location, output, android, fetch_license, fetch_license_djc, reference,
 
 def validate_template(ctx, param, value):
     if not value:
-        return DEFAULT_TEMPLATE_FILE
+        return None
 
     with io.open(value, encoding='utf-8') as templatef:
         template_error = check_template(templatef.read())
@@ -383,7 +383,7 @@ def attrib(input, output, api_url, api_key, scancode, min_license_score, referen
             msg = 'The input file from scancode toolkit needs to be in JSON format.'
             click.echo(msg)
             sys.exit(1)
-        if not min_license_score:
+        if not min_license_score and not min_license_score == 0:
             min_license_score=DEFAULT_LICENSE_SCORE
 
     if min_license_score:
@@ -452,6 +452,7 @@ def attrib(input, output, api_url, api_key, scancode, min_license_score, referen
             is_about_input=is_about_input,
             license_dict=dict(sorted(license_dict.items())),
             output_location=output,
+            scancode=scancode,
             min_license_score=min_license_score,
             template_loc=template,
             variables=vartext,

src/attributecode/gen.py

@@ -211,6 +211,7 @@ def load_inventory(location, from_attrib=False, base_dir=None, scancode=False, r
         ld_errors = about.load_dict(
             fields,
             base_dir,
+            scancode=scancode,
             from_attrib=from_attrib,
             running_inventory=False,
             reference_dir=reference_dir,
@@ -237,6 +238,7 @@ def load_inventory(location, from_attrib=False, base_dir=None, scancode=False, r
                 about.license_score.value = score_list
             except:
                 pass
+
     return unique(errors), abouts
 
 

src/attributecode/model.py

@@ -944,7 +944,7 @@ def hydrate(self, fields):
         return errors
 
     def process(self, fields, about_file_path, running_inventory=False,
-                base_dir=None, from_attrib=False, reference_dir=None):
+                base_dir=None, scancode=False, from_attrib=False, reference_dir=None):
         """
         Validate and set as attributes on this About object a sequence of
         `fields` name/value tuples. Return a list of errors.
@@ -961,13 +961,18 @@ def process(self, fields, about_file_path, running_inventory=False,
             errors.extend(copy_err)
 
         # TODO: why? we validate all fields, not only these hydrated
-        validation_errors = validate_fields(
-            self.all_fields(),
-            about_file_path,
-            running_inventory,
-            self.base_dir,
-            self.reference_dir)
-        errors.extend(validation_errors)
+        # The validate functions does not allow duplicated entry for a list meaning
+        # it will cause problem when using scancode license detection as an input as
+        # it usually returns duplicated license_key and many license have duplicated
+        # score such as 100. We need to handle this scenario using different method.
+        if not scancode:
+            validation_errors = validate_fields(
+                self.all_fields(),
+                about_file_path,
+                running_inventory,
+                self.base_dir,
+                self.reference_dir)
+            errors.extend(validation_errors)
         return errors
 
     def load(self, location):
@@ -1015,7 +1020,7 @@ def load(self, location):
 
     # FIXME: should be a from_dict class factory instead
     # FIXME: running_inventory: remove this : this should be done in the commands, not here
-    def load_dict(self, fields_dict, base_dir, from_attrib=False, running_inventory=False, reference_dir=None,):
+    def load_dict(self, fields_dict, base_dir, scancode=False, from_attrib=False, running_inventory=False, reference_dir=None,):
         """
         Load this About object file from a `fields_dict` name/value dict.
         Return a list of errors.
@@ -1046,14 +1051,17 @@ def load_dict(self, fields_dict, base_dir, from_attrib=False, running_inventory=
                 # 'Field licenses is a custom field.'
                 licenses_field = (key, value)
                 fields.remove(licenses_field)
+
         errors = self.process(
             fields=fields,
             about_file_path=self.about_file_path,
             running_inventory=running_inventory,
             base_dir=base_dir,
+            scancode=scancode,
             from_attrib=from_attrib,
             reference_dir=reference_dir,
         )
+
         self.errors = errors
         return errors
 
@@ -1570,20 +1578,10 @@ def pre_process_and_fetch_license_dict(abouts, api_url=None, api_key=None, scanc
         if scancode:
             lic_exp = ''
             lic_list = []
-            # Since the model treats license_expressions (from scancode scan) as a custom field
-            # in string format, we need to capture this string to convert to a list
-            # and then use the `AND` condition if multiple licenses exist.
-            # See https://github.com/nexB/aboutcode-toolkit/issues/479#issuecomment-946328428 
+            # The license_expressions return from scancode is a list of license keys.
+            # Therefore, we will combine it with the 'AND' condition
             if about.license_expressions.value:
-                # Stripping '[', ']', quote and spaces
-                converted_lic_exp = about.license_expressions.value.strip("[").strip("]").replace('\'','').replace(' ','')
-                # Convert the updated lic_exp string to list
-                converted_lic_list = converted_lic_exp.split(',')
-                for lic in converted_lic_list:
-                    # Only keep unique license keys
-                    if not lic in lic_list:
-                        lic_list.append(lic)
-            lic_exp = " AND ".join(lic_list)
+                lic_exp = " AND ".join(about.license_expressions.value)
             about.license_expression.value = lic_exp
             about.license_expression.present = True
 

templates/scancode_html.template

@@ -18,94 +18,50 @@
     </div>
 
         <div class="oss-table-of-contents">
-			{% set index = namespace(value=0) %}
+            {% set index = namespace(value=0) %}
             {% for about_object in abouts %}
-				{% set captured = {} %}
-				{% if about_object.license_expression.value %}
-					{% for lic_score in about_object.license_score.value %}
-						{% if lic_score | float >= min_license_score %}
-							{% if not captured[about_object.name.value] %}
-								<p><a href="#component_{{ index.value }}">{{ about_object.name.value }}{% if about_object.version.value %} {{ about_object.version.value }}{% endif %}</a></p>
-								{% set _ = captured.update({ about_object.name.value: true }) %}
-								{% set index.value = index.value + 1 %}
-							{% endif %}
-						{% endif %}
-					{% endfor %}
-				{% endif %}
+                {% set captured = {} %}
+                {% if about_object.license_key.value %}
+					{% if not captured[about_object.name.value] %}
+						<p><a href="#component_{{ index.value }}">{{ about_object.name.value }}{% if about_object.version.value %} {{ about_object.version.value }}{% endif %}</a></p>
+						{% set _ = captured.update({ about_object.name.value: true }) %}
+						{% set index.value = index.value + 1 %}
+					{% endif %}
+                {% endif %}
             {% endfor %}
         </div>
 
     <hr/>
 
-	{% set common_licenses_meet_score = {} %}
-	{% set index = namespace(value=0) %}
+    {% set index = namespace(value=0) %}
     {% for about_object in abouts %}
-		{% set captured = {} %}
-        {% if about_object.license_expression.value %}
-			{% set count = namespace(value=0) %}
-			{{ about_object.license_key.value }}
-			{{ about_object.license_score.value }}
-			{% for lic_score in about_object.license_score.value %}
-				{% if lic_score | float >= min_license_score %}
-					{% if not captured[about_object.name.value] %}
-						<div class="oss-component" id="component_{{ index.value }}">
-						<h3 class="component-name">{{ about_object.name.value }} {% if about_object.version.value %}{{ about_object.version.value }}{% endif %} </h3>
-						{% set _ = captured.update({ about_object.name.value: true }) %}
-						{% set index.value = index.value + 1 %}
-					{% endif %}
-					<p>This component is licensed under {{ about_object.license_name.value[count.value] }}</p>
-				{% endif %}
-				{% set count.value = count.value + 1 %}
-			{% endfor %}
-			{% if about_object.copyright.value %}
-				<pre>{{about_object.copyright.value}}</pre>
-			{% endif %}
-			{% if about_object.notice_file.value %}
-				{% for notice in about_object.notice_file.value %}
-					<pre class="component-notice">{{ about_object.notice_file.value[notice] }}</pre>
-				{% endfor %}
+        {% set captured = {} %}
+        {% if about_object.license_key.value %}
+			{% if not captured[about_object.name.value] %}
+				<div class="oss-component" id="component_{{ index.value }}">
+				<h3 class="component-name">{{ about_object.name.value }} {% if about_object.version.value %}{{ about_object.version.value }}{% endif %} </h3>
+				{% set _ = captured.update({ about_object.name.value: true }) %}
+				{% set index.value = index.value + 1 %}
 			{% endif %}
-			{% if about_object.license_key.value %}
-				{% if about_object.license_file.value %}
-					{% for lic_file_name in about_object.license_file.value %}
-						{% for license in licenses_list %}
-							{% if license.filename == lic_file_name %}
-								{% if not license.key in common_licenses %}
-									<pre> {{ license.text | e}} </pre>
-								{% endif %}
-							{% endif %}
-						{% endfor %}
-					{% endfor %}
+            {% if about_object.copyright.value %}
+                <pre>{{about_object.copyright.value}}</pre>
+            {% endif %}
+
+
+			{% for lic_key in about_object.license_key.value %}
+				<p>This component is licensed under {{ lic_key }}</p>
+				{% if lic_key in common_licenses %}
+					<p>Full text of <a class="{{ lic_key }}" href="#component-license-{{ lic_key }}"> {{ lic_key }}</a> is available at the end of this document.</p>
 				{% else %}
-					{% set count = namespace(value=0) %}
-					{% for lic_score in about_object.license_score.value %}
-						{% if lic_score | float >= min_license_score %}
-							{% if about_object.license_key.value[count.value] in common_licenses %}
-								{% if not about_object.license_key.value[count.value] in common_licenses_meet_score %}
-									{% set _ = common_licenses_meet_score.update({ about_object.license_key.value[count.value]: true }) %}
-									<p>Full text of <a class="{{ about_object.license_key.value[count.value] }}" href="#component-license-{{ about_object.license_key.value[count.value] }}"> {{ about_object.license_key.value[count.value] }}</a> is available at the end of this document.</p>
-								{% endif %}
-							{% else %}
-								{% for license in licenses_list %}
-									{% if about_object.license_key.value[count.value] == license.key %}
-										<h3 id="component-license-{{ license.key }}">{{ license.key }}</h3>
-										<pre> {{ license.text | e }} </pre>
-									{% endif %}
-								{% endfor %}
-							{% endif %}
-						{% endif %}
-					{% endfor %}
-				{% endif %}
-			{% else %}
-				{% if about_object.license_file.value %}
-					{% for lic_file_name in about_object.license_file.value %}
-						{% if about_object.license_file.value[lic_file_name] %}
-							<pre> {{ about_object.license_file.value[lic_file_name] | e}} </pre>
+					{% for license in licenses_list %}
+						{% if lic_key == license.key %}
+							<h3 id="component-license-{{ license.key }}">{{ license.key }}</h3>
+							<pre> {{ license.text | e }} </pre>
 						{% endif %}
 					{% endfor %}
 				{% endif %}
-			{% endif %}
-		{% endif %}
+			{% endfor %}
+        {% endif %}
         </div>
     {% endfor %}
 
@@ -114,10 +70,8 @@
     <h3>Common Licenses Used in This Product</h3>
     {% for license in licenses_list %}
         {% if license.key in common_licenses %}
-			{% if license.key in common_licenses_meet_score %}
-				<h3 id="component-license-{{ license.key }}">{{ license.key }}</h3>
-				<pre> {{ license.text | e }} </pre>
-			{% endif %}
+			<h3 id="component-license-{{ license.key }}">{{ license.key }}</h3>
+			<pre> {{ license.text | e }} </pre>
         {% endif %}
     {% endfor %}