Merge pull request #3402 from nexB/fix-extracted-license-statement

Format extracted_license_statement as YAML

Author: AyanSinhaMahapatra

CHANGELOG.rst

@@ -135,7 +135,9 @@ License detection:
     These expressions are parallel to detections.
 
   - The ``declared_license`` attribute is renamed ``extracted_license_statement``
-    and is now a YAML-encoded string.
+    and is now a YAML-encoded string, which can be parsed to recreate the
+    original extracted license statement. Previously this used to be nested
+    python objects lists/dicts/string, but now this is always a YAML string.
 
     See `license updates documentation <https://scancode-toolkit.readthedocs.io/en/latest/explanations/license-detection-reference.html#change-in-license-data-format-package>`_
     for examples and details.

src/packagedcode/models.py

@@ -16,6 +16,7 @@
 import attr
 from packageurl import normalize_qualifiers
 from packageurl import PackageURL
+import saneyaml
 
 from commoncode import filetype
 from commoncode.datautils import choices
@@ -780,7 +781,7 @@ def populate_license_fields(self):
                 )
 
         if self.extracted_license_statement and not isinstance(self.extracted_license_statement, str):
-            self.extracted_license_statement = repr(self.extracted_license_statement)
+            self.extracted_license_statement = saneyaml.dump(self.extracted_license_statement)
 
     def to_dict(self, with_details=True, **kwargs):
         mapping = super().to_dict(with_details=with_details, **kwargs)

src/scancode/cli_test_utils.py

@@ -340,6 +340,7 @@ def check_jsonlines_scan(
     regen=False,
     remove_file_date=False,
     check_headers=False,
+    remove_uuid=True,
 ):
     """
     Check the scan result_file JSON Lines results against the expected_file
@@ -352,6 +353,9 @@ def check_jsonlines_scan(
     with io.open(result_file, encoding='utf-8') as res:
         results = [json.loads(line) for line in res]
 
+    if remove_uuid:
+        for result in results:
+            result = remove_uuid_from_scan(result)
     streamline_jsonlines_scan(results, remove_file_date)
 
     if regen:
@@ -360,6 +364,9 @@ def check_jsonlines_scan(
 
     with io.open(expected_file, encoding='utf-8') as res:
         expected = json.load(res)
+        if remove_uuid:
+            for result in results:
+                result = remove_uuid_from_scan(result)
 
     streamline_jsonlines_scan(expected, remove_file_date)