Merge pull request #3220 from nexB/get-valid-yaml

Get valid yaml output

Author: pombredanne

CHANGELOG.rst

@@ -190,6 +190,12 @@ License detection:
   This is now built-in with the --license-text option and --info
   and exposed with the "percentage_of_license_text" attribute.
 
+- The license dump() has been modified to add an extra space at empty
+  newlines for license files which also have multiple indentation levels
+  as this was generating invalid YAML output files when ``--license-text``
+  or ``--license-references`` was enabled.
+
+  See https://github.com/nexB/scancode-toolkit/issues/3219
 
 v31.2.4 - 2023-01-09
 ----------------------------------

docs/source/cli-reference/other-commands.rst

@@ -100,3 +100,11 @@ reindex the licenses without these licenses from the additional plugins.
 
 Rebuild the license index including texts all languages (and not only
 English) and exit. This is an EXPERIMENTAL option.
+
+
+``--load-dump`` Option
+^^^^^^^^^^^^^^^^^^^^^^
+
+Load all licenses and rules from their respective files and then dump them
+to their respective files. This is done to make small formatting changes across
+all licenses and rules, to be consistent across them.

docs/source/rst_snippets/scancode-reindex-licenses.rst

@@ -18,4 +18,6 @@ Options
   --additional-directory DIR  Include this directory with additional custom
                               licenses and license rules in the license
                               detection index.
+  --load-dump                 Load all license and rules from their respective
+                              files and then dump them back to those same files.
   -h, --help                  Shows the options and explanations.

etc/scripts/licenses/buildrules.py

@@ -213,7 +213,7 @@ def cli(licenses_file):
         rl = models.BasicRule(text=rdata.text, **rdata.data)
         skinny_rules.append(rl)
 
-    models.validate_rules(skinny_rules, licenses_by_key, with_text=True)
+    models.validate_rules(skinny_rules, licenses_by_key, with_text=True, thorough=True)
 
     print()
     for rule in skinny_rules:

src/licensedcode/cache.py

@@ -29,7 +29,7 @@
 # global in-memory cache of the LicenseCache
 _LICENSE_CACHE = None
 
-LICENSE_INDEX_LOCK_TIMEOUT = 60 * 4
+LICENSE_INDEX_LOCK_TIMEOUT = 60 * 6
 LICENSE_INDEX_DIR = 'license_index'
 LICENSE_INDEX_FILENAME = 'index_cache'
 LICENSE_LOCKFILE_NAME = 'scancode_license_index_lockfile'
@@ -134,8 +134,6 @@ def load_or_build(
         try:
             # acquire lock and wait until timeout to get a lock or die
             with lockfile.FileLock(lock_file).locked(timeout=timeout):
-                # Here, the cache is either stale or non-existing: we need to
-                # rebuild all cached data (e.g. mostly the index) and cache it
 
                 additional_directories = []
                 if only_builtin:
@@ -156,6 +154,8 @@ def load_or_build(
                     additional_directories=additional_license_dirs,
                     scancode_license_dir=licenses_data_dir
                 )
+                # Here, the cache is either stale or non-existing: we need to
+                # rebuild all cached data (e.g. mostly the index) and cache it
                 licenses_db = load_licenses_from_multiple_dirs(
                     additional_license_data_dirs=additional_license_dirs,
                     builtin_license_data_dir=licenses_data_dir,