Merge pull request #3275 from nexB/fix-html-output

Fix licenses in HTML output

Author: AyanSinhaMahapatra

src/formattedcode/output_html.py

@@ -7,6 +7,8 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 import io
+import os
+import logging
 from operator import itemgetter
 from os.path import abspath
 from os.path import dirname
@@ -41,6 +43,24 @@
 TEMPLATES_DIR = join(dirname(__file__), 'templates')
 
 
+TRACE = False
+
+
+def logger_debug(*args):
+    pass
+
+
+logger = logging.getLogger(__name__)
+
+if TRACE:
+    import sys
+    logging.basicConfig(stream=sys.stdout)
+    logger.setLevel(logging.DEBUG)
+
+    def logger_debug(*args):
+        return logger.debug(' '.join(isinstance(a, str) and a or repr(a) for a in args))
+
+
 @output_impl
 class HtmlOutput(OutputPlugin):
 
@@ -59,9 +79,18 @@ def is_enabled(self, html, **kwargs):
     def process_codebase(self, codebase, html, **kwargs):
         results = self.get_files(codebase, **kwargs)
         version = codebase.get_or_create_current_header().tool_version
+        license_references = []
+        if hasattr(codebase.attributes, 'license_references'):
+            license_references = codebase.attributes.license_references
         template_loc = join(TEMPLATES_DIR, 'html', 'template.html')
         output_file = html
-        write_templated(output_file, results, version, template_loc)
+        write_templated(
+            output_file=output_file,
+            results=results,
+            license_references=license_references,
+            version=version,
+            template_loc=template_loc,
+        )
 
 
 @output_impl
@@ -98,20 +127,34 @@ def is_enabled(self, custom_output, custom_template, **kwargs):
     def process_codebase(self, codebase, custom_output, custom_template, **kwargs):
         results = self.get_files(codebase, **kwargs)
         version = codebase.get_or_create_current_header().tool_version
+        license_references = []
+        if hasattr(codebase.attributes, 'license_references'):
+            license_references = codebase.attributes.license_references
         template_loc = custom_template
         output_file = custom_output
-        write_templated(output_file, results, version, template_loc)
+        write_templated(
+            output_file=output_file,
+            results=results,
+            license_references=license_references,
+            version=version,
+            template_loc=template_loc
+        )
 
 
-def write_templated(output_file, results, version, template_loc):
+def write_templated(output_file, results, license_references, version, template_loc):
     """
     Write scan output `results` to the `output_file` opened file using a template
     file at `template_loc`.
     Raise an exception on errors.
     """
     template = get_template(template_loc)
 
-    for template_chunk in generate_output(results, version, template):
+    for template_chunk in generate_output(
+        results=results,
+        license_references=license_references,
+        version=version,
+        template=template,
+    ):
         assert isinstance(template_chunk, str)
         try:
             output_file.write(template_chunk)
@@ -138,23 +181,18 @@ def get_template(location):
     return env.get_template(template_name)
 
 
-def generate_output(results, version, template):
+def generate_output(results, license_references, version, template):
     """
     Yield unicode strings from incrementally rendering `results` and `version`
     with the Jinja `template` object.
     """
     # FIXME: This code is highly coupled with actual scans and may not
     # support adding new scans at all
-
-    from licensedcode.cache import get_licenses_db
-    licenses_db = get_licenses_db()
-
     converted = {}
     converted_infos = {}
     converted_packages = {}
-    licenses = {}
 
-    LICENSES = 'licenses'
+    LICENSES = 'license_detections'
     COPYRIGHTS = 'copyrights'
     PACKAGES = 'package_data'
 
@@ -175,6 +213,8 @@ def generate_output(results, version, template):
             for match in get_matches_from_detection_mappings(scanned_file[LICENSES]):
                 # make copy
                 match = dict(match)
+                if TRACE:
+                    logger_debug(f"match: {match}")
                 license_expression = match['license_expression']
                 results.append({
                     'start': match['start_line'],
@@ -183,11 +223,6 @@ def generate_output(results, version, template):
                     'value': license_expression,
                 })
 
-                # FIXME: we should NOT rely on license objects: only use what is in the JSON instead
-                if license_expression not in licenses:
-                    licenses[license_expression] = match
-                    # we were modifying the scan data in place ....
-                    match['object'] = licenses_db.get(license_expression)
         if results:
             converted[path] = sorted(results, key=itemgetter('start'))
 
@@ -204,15 +239,13 @@ def generate_output(results, version, template):
         if PACKAGES in scanned_file:
             converted_packages[path] = scanned_file[PACKAGES]
 
-        licenses = dict(sorted(licenses.items()))
-
     files = {
         'license_copyright': converted,
         'infos': converted_infos,
         'package_data': converted_packages
     }
 
-    return template.generate(files=files, licenses=licenses, version=version)
+    return template.generate(files=files, license_references=license_references, version=version)
 
 
 @output_impl
@@ -224,6 +257,7 @@ class HtmlAppOutput(OutputPlugin):
         PluggableCommandLineOption(('--html-app',),
             type=FileOptionType(mode='w', encoding='utf-8', lazy=True),
             metavar='FILE',
+            hidden=True,
             help='(DEPRECATED: use the ScanCode Workbench app instead ) '
                   'Write scan output as a mini HTML application to FILE.',
             help_group=OUTPUT_GROUP,

src/formattedcode/templates/html/template.html

@@ -259,36 +259,42 @@
     </table>
     {% endif %}
 
-    {% if licenses %}
+    {% if license_references %}
     <table>
-      <caption>Licenses</caption>
+      <caption>License References</caption>
       <thead>
         <tr>
           <th>key</th>
           <th>short_name</th>
           <th>category</th>
           <th>owner</th>
-          <th>reference_url</th>
+          <th>scancode_url</th>
+          <th>licensedb_url</th>
           <th>homepage_url</th>
-          <th>text_url</th>
+          <th>text_urls</th>
           <th>spdx_license_key</th>
           <th>spdx_url</th>
         </tr>
       </thead>
       <tbody>
-        {% for key, license in licenses.items() %}
-          <tr id="license_{{ license.key }}">
-            <td>{{ license.key }}</td>
-            <td>{{ license.short_name }}</td>
-            <td>{{ license.category }}</td>
-            <td>{{ license.owner }}</td>
-            <td>{{ license.reference_url|urlize(target='_blank') }}</td>
-            <td>{{ license.homepage_url|urlize(target='_blank') }}</td>
-            <td>{{ license.text_url|urlize(target='_blank') }}</td>
-            <td>{{ license.spdx_license_key }}</td>
-            <td>{{ license.spdx_url|urlize(target='_blank') }}</td>
-          </tr>
-        {% endfor %}
+      {% for license_reference in license_references %}
+        <tr id="license_{{ license_reference.key }}">
+          <td>{{ license_reference.key }}</td>
+          <td>{{ license_reference.short_name }}</td>
+          <td>{{ license_reference.category }}</td>
+          <td>{{ license_reference.owner }}</td>
+          <td>{{ license_reference.scancode_url|urlize(target='_blank') }}</td>
+          <td>{{ license_reference.licensedb_url|urlize(target='_blank') }}</td>
+          <td>{{ license_reference.homepage_url|urlize(target='_blank') }}</td>
+          <td>
+            {% for text_url in license_reference.text_urls %}
+              {{ text_url|urlize(target='_blank') }}
+            {% endfor %}
+          </td>
+          <td>{{ license_reference.spdx_license_key }}</td>
+          <td>{{ license_reference.spdx_url|urlize(target='_blank') }}</td>
+        </tr>
+      {% endfor %}
       </tbody>
     </table>
     {% endif %}

tests/formattedcode/data/templated/simple-license.txt

@@ -0,0 +1 @@
+license: gpl-2.0

tests/formattedcode/test_output_templated.py

@@ -29,6 +29,26 @@
 test_env.test_data_dir = os.path.join(os.path.dirname(__file__), 'data')
 
 
+@pytest.mark.scanslow
+def test_has_licenses_in_html_format_output():
+    test_file = test_env.get_test_loc('templated/simple-license.txt')
+    result_file = test_env.get_temp_file('html')
+    run_scan_click(['--license', test_file, '--html', result_file])
+    results = open(result_file).read()
+    assert 'gpl-2.0' in results
+    assert __version__ in results
+
+
+@pytest.mark.scanslow
+def test_has_license_references_in_html_format_output():
+    test_file = test_env.get_test_loc('templated/simple-license.txt')
+    result_file = test_env.get_temp_file('html')
+    run_scan_click(['--license', '--license-references', test_file, '--html', result_file])
+    results = open(result_file).read()
+    assert 'http://www.gnu.org/licenses/gpl-2.0.html' in results
+    assert __version__ in results
+
+
 @pytest.mark.scanslow
 def test_paths_are_posix_paths_in_html_app_format_output():
     test_dir = test_env.get_test_loc('templated/simple')

tests/scancode/data/help/help.txt

@@ -61,8 +61,6 @@ Options:
     --cyclonedx-xml FILE    Write scan output in CycloneDX XML format to FILE.
     --spdx-rdf FILE         Write scan output as SPDX RDF to FILE.
     --spdx-tv FILE          Write scan output as SPDX Tag/Value to FILE.
-    --html-app FILE         (DEPRECATED: use the ScanCode Workbench app instead )
-                            Write scan output as a mini HTML application to FILE.
 
   output filters:
     --ignore-author <pattern>       Ignore a file (and all its findings) if an