Merge pull request #3197 from nexB/fix-license-dump

Update license db generation

Author: pombredanne

CHANGELOG.rst

@@ -175,7 +175,7 @@ License detection:
 
   - See https://github.com/nexB/scancode-toolkit/issues/3049
 
-- There is a new ``--get-license-data`` scancode command line option to export
+- There is a new console script ``scancode-license-data`` to export
   license data in JSON, YAML and HTML, with indexes and a static website for use
   in the licensedb web site. This becomes the  API way to getr scancode license
   data.

setup-mini.cfg

@@ -147,6 +147,7 @@ packages =
 console_scripts =
     scancode = scancode.cli:scancode
     scancode-reindex-licenses = licensedcode.reindex:reindex_licenses
+    scancode-license-data = licensedcode.license_db:dump_scancode_license_data
 
 # These are configurations for ScanCode plugins as setuptools entry points.
 # Each plugin entry hast this form:

setup.cfg

@@ -147,6 +147,7 @@ packages =
 console_scripts =
     scancode = scancode.cli:scancode
     scancode-reindex-licenses = licensedcode.reindex:reindex_licenses
+    scancode-license-data = licensedcode.license_db:dump_scancode_license_data
 
 # These are configurations for ScanCode plugins as setuptools entry points.
 # Each plugin entry hast this form:

src/licensedcode/cache.py

@@ -182,6 +182,12 @@ def load_or_build(
             # TODO: handle unable to lock in a nicer way
             raise
 
+    @property
+    def has_additional_licenses(self):
+        cache = get_cache()
+        if cache.additional_license_directory or cache.additional_license_plugins:
+            return True
+
 
 def build_index(
     licenses_db=None,

src/licensedcode/license_db.py

@@ -31,7 +31,11 @@
 from os.path import join
 from distutils.dir_util import copy_tree
 
+import click
 import saneyaml
+
+from commoncode.cliutils import MISC_GROUP
+from commoncode.cliutils import PluggableCommandLineOption
 from jinja2 import Environment, FileSystemLoader
 from licensedcode.models import load_licenses
 from licensedcode.models import licenses_data_dir
@@ -126,13 +130,16 @@ def generate_details(output_path, environment, licenses, test=False):
 
     ``test`` is to generate a stable output for testing only
     """
+    from licensedcode.cache import get_cache
+    include_builtin = get_cache().has_additional_licenses
+
     if test:
         base_context_mapping = base_context_test
     else:
         base_context_mapping = base_context
     license_details_template = environment.get_template("license_details.html")
     for lic in licenses.values():
-        license_data = lic.to_dict(include_text=True)
+        license_data = lic.to_dict(include_text=False, include_builtin=include_builtin)
         html = license_details_template.render(
             **base_context_mapping,
             license=lic,
@@ -200,19 +207,40 @@ def generate(
     return count
 
 
-def dump_license_data(ctx, param, value):
+def scancode_license_data(path):
     """
-    Dump license data from scancode licenses to the directory ``value`` passed
+    Dump license data from scancode licenses to the directory ``path`` passed
     in from command line.
 
     Dumps data in JSON, YAML and HTML formats and also dumps the .LICENSE file
     with the license text and the data as YAML frontmatter.
     """
-    if not value or ctx.resilient_parsing:
-        return
-
-    import click
-    click.secho(f'Dumping license data to: {value}', err=True)
-    count = generate(build_location=value)
+    click.secho(f'Dumping license data to: {path}', err=True)
+    count = generate(build_location=path)
     click.secho(f'Done dumping #{count} licenses.', err=True)
-    ctx.exit(0)
+
+
+@click.command(name='scancode-license-data')
+@click.option(
+    '--path',
+    type=click.Path(exists=False, writable=True, file_okay=False, resolve_path=True, path_type=str),
+    metavar='DIR',
+    help='Dump the license data in this directory in the LicenseDB format and exit. '
+            'Creates the directory if it does not exist. ',
+    help_group=MISC_GROUP,
+    cls=PluggableCommandLineOption,
+)
+@click.help_option('-h', '--help')
+def dump_scancode_license_data(
+    path,
+    *args,
+    **kwargs,
+):
+    """
+    Dump scancode license data in various formats, and the licenseDB static website at `path`.
+    """
+    scancode_license_data(path=path)
+
+
+if __name__ == '__main__':
+    dump_scancode_license_data()

src/licensedcode/plugin_license.py

@@ -12,11 +12,9 @@
 from functools import partial
 
 import attr
-import click
 from commoncode.cliutils import PluggableCommandLineOption
 from commoncode.cliutils import SCAN_GROUP
 from commoncode.cliutils import SCAN_OPTIONS_GROUP
-from commoncode.cliutils import MISC_GROUP
 from plugincode.scan import ScanPlugin
 from plugincode.scan import scan_impl
 
@@ -33,7 +31,6 @@
 from licensedcode.detection import LicenseDetectionFromResult
 from licensedcode.detection import LicenseMatchFromResult
 from licensedcode.detection import UniqueDetection
-from licensedcode.license_db import dump_license_data
 from packagedcode.utils import combine_expressions
 from scancode.api import SCANCODE_LICENSEDB_URL
 
@@ -121,19 +118,7 @@ class LicenseScanner(ScanPlugin):
             required_options=['license'],
             help='[EXPERIMENTAL] Detect unknown licenses. ',
             help_group=SCAN_OPTIONS_GROUP,
-        ),
-
-        # TODO: consider creating a separate comamnd line option exe instead
-        PluggableCommandLineOption(
-            ('--dump-license-data',),
-            type=click.Path(exists=False, readable=True, file_okay=False, resolve_path=True, path_type=str),
-            metavar='DIR',
-            callback=dump_license_data,
-            help='Dump the license data in this directory in the LicenseDB format and exit. '
-            'Creates the directory if it does not exist. ',
-            help_group=MISC_GROUP,
-            is_eager=True,
-        ),
+        )
     ]
 
     def is_enabled(self, license, **kwargs):  # NOQA
@@ -176,20 +161,17 @@ def process_codebase(self, codebase, **kwargs):
         cche = cache.get_cache()
 
         cle = codebase.get_or_create_current_header()
-        has_additional_licenses = False
 
         if cche.additional_license_directory:
             cle.extra_data['additional_license_directory'] = cche.additional_license_directory
-            has_additional_licenses = True
 
         if cche.additional_license_plugins:
             cle.extra_data['additional_license_plugins'] = cche.additional_license_plugins
-            has_additional_licenses = True
 
-        if TRACE and has_additional_licenses:
+        if TRACE and cche.has_additional_licenses:
             logger_debug(
                 f'add_referenced_filenames_license_matches: additional_licenses',
-                f'has_additional_licenses: {has_additional_licenses}\n',
+                f'has_additional_licenses: {cche.has_additional_licenses}\n',
                 f'additional_license_directory: {cche.additional_license_directory}\n',
                 f'additional_license_plugins : {cche.additional_license_plugins}'
             )

src/licensedcode/templates/footer.html

@@ -8,5 +8,6 @@
     </p>
     <p>Designed and built by <a href="https://www.nexb.com/" target="_blank">nexB</a>. Licensed under the <a href="cc-by-4.0.html">Creative Commons Attribution License 4.0 (CC-BY-4.0)</a>.</p>
     <p>Generated with <a href="https://github.com/nexB/scancode-toolkit" target="_blank">ScanCode toolkit</a> {{ scancode_version }} on {{ now }}.</p>
+    <p>This is updated daily automatically with latest updates from the develop branch of scancode-toolkit, if any.</p>
   </footer>
 </div>

tests/licensedcode/data/license_db/license_dump/bash-exception-gpl.html

@@ -106,13 +106,6 @@
 
         </dd>
 
-        <dt style="">is_builtin</dt>
-        <dd>
-          
-            True
-          
-        </dd>
-      
         <dt style="">is_exception</dt>
         <dd>
 
@@ -136,19 +129,6 @@
 
         </dd>
 
-        <dt style="">text</dt>
-        <dd>
-          
-            The Free Software Foundation has exempted Bash from the requirement of
-Paragraph 2c of the General Public License.  This is to say, there is
-no requirement for Bash to print a notice when it is started
-interactively in the usual way.  We made this exception because users
-and standards expect shells not to print such messages.  This
-exception applies to any program that serves as a shell and that is
-based primarily on Bash as opposed to other GNU software.
-          
-        </dd>
-      
     </dl>
     <div class="text-bold">license_text</div>
     <pre id="license-text" class="code mt-1" style="white-space: pre-wrap;"><code>The Free Software Foundation has exempted Bash from the requirement of

tests/licensedcode/data/license_db/license_dump/bash-exception-gpl.json

@@ -5,11 +5,9 @@
   "category": "Copyleft",
   "owner": "Free Software Foundation (FSF)",
   "notes": "this used with GPL 1.0 and 2.0. It was removed from the V3 text https://git.savannah.gnu.org/cgit/bash.git/commit/COPYING?id=3185942a5234e26ab13fa02f9c51d340cec514f8",
-  "is_builtin": true,
   "is_exception": true,
   "spdx_license_key": "LicenseRef-scancode-bash-exception-gpl-2.0",
   "text_urls": [
     "https://git.savannah.gnu.org/cgit/bash.git/plain/COPYING?h=bash-3.0-rc1&id=dd9e6dfa23d0dae4888f11fb8c6a27bc36d1b283"
-  ],
-  "text": "The Free Software Foundation has exempted Bash from the requirement of\nParagraph 2c of the General Public License.  This is to say, there is\nno requirement for Bash to print a notice when it is started\ninteractively in the usual way.  We made this exception because users\nand standards expect shells not to print such messages.  This\nexception applies to any program that serves as a shell and that is\nbased primarily on Bash as opposed to other GNU software."
+  ]
 }

tests/licensedcode/data/license_db/license_dump/bash-exception-gpl.yml

@@ -4,16 +4,7 @@ name: Bash exception to GPL
 category: Copyleft
 owner: Free Software Foundation (FSF)
 notes: this used with GPL 1.0 and 2.0. It was removed from the V3 text https://git.savannah.gnu.org/cgit/bash.git/commit/COPYING?id=3185942a5234e26ab13fa02f9c51d340cec514f8
-is_builtin: yes
 is_exception: yes
 spdx_license_key: LicenseRef-scancode-bash-exception-gpl-2.0
 text_urls:
   - https://git.savannah.gnu.org/cgit/bash.git/plain/COPYING?h=bash-3.0-rc1&id=dd9e6dfa23d0dae4888f11fb8c6a27bc36d1b283
-text: |
-  The Free Software Foundation has exempted Bash from the requirement of
-  Paragraph 2c of the General Public License.  This is to say, there is
-  no requirement for Bash to print a notice when it is started
-  interactively in the usual way.  We made this exception because users
-  and standards expect shells not to print such messages.  This
-  exception applies to any program that serves as a shell and that is
-  based primarily on Bash as opposed to other GNU software.