Merge pull request #3299 from nexB/license-detection-follow-up

Fix misc license detection related bugs

Author: AyanSinhaMahapatra

sample.json

@@ -522,7 +522,7 @@ def detections_from_license_detection_mappings(
     return license_detections
 
 
-def get_new_identifier_from_detections(initial_detection, detections_added):
+def get_new_identifier_from_detections(initial_detection, detections_added, license_expression):
     """
     Return a new UUID based on two sets of detections: `initial_detection` is
     the detection being modified with a list of detections (from another file region)
@@ -533,7 +533,8 @@ def get_new_identifier_from_detections(initial_detection, detections_added):
         for detection_mapping in detections_added
     ]
     identifiers.append(initial_detection["identifier"])
-    return get_uuid_on_content(content=sorted(identifiers))
+    uuid = get_uuid_on_content(content=sorted(identifiers))
+    return f"{license_expression}-{uuid}"
 
 
 @attr.s

src/licensedcode/detection.py

@@ -354,6 +354,7 @@ def add_referenced_filenames_license_matches_for_detections(resource, codebase):
         license_detection_mapping["identifier"] = get_new_identifier_from_detections(
             initial_detection=license_detection_mapping,
             detections_added=detections_added,
+            license_expression=license_expression,
         )
 
     if modified:

src/licensedcode/plugin_license.py

@@ -49,7 +49,7 @@ class LicensePolicy(PostScanPlugin):
     detected license key that is found in the license_policy.yml file
     """
 
-    resource_attributes = dict(license_policy=attr.ib(default=attr.Factory(dict)))
+    resource_attributes = dict(license_policy=attr.ib(default=attr.Factory(list)))
 
     sort_order = 9
 
@@ -90,16 +90,19 @@ def process_codebase(self, codebase, license_policy, **kwargs):
 
             except AttributeError:
                 # add license_policy regardless if there is license info or not
-                resource.license_policy = {}
+                resource.license_policy = []
                 codebase.save_resource(resource)
                 continue
 
+            license_policies = []
             for key in resource_license_keys:
                 for policy in policies:
                     if key == policy.get('license_key'):
                         # Apply the policy to the Resource
-                        resource.license_policy = policy
-                        codebase.save_resource(resource)
+                        license_policies.append(policy)
+            
+            resource.license_policy = sorted(license_policies, key=lambda d: d['license_key']) 
+            codebase.save_resource(resource)
 
 
 def has_policy_duplicates(license_policy_location):

src/licensedcode/plugin_license_policy.py

@@ -136,6 +136,7 @@ def add_referenced_license_matches_for_package(resource, codebase, no_licenses):
             license_detection_mapping["identifier"] = get_new_identifier_from_detections(
                 initial_detection=license_detection_mapping,
                 detections_added=referenced_license_detections,
+                license_expression=license_expression,
             )
 
         if modified:
@@ -252,6 +253,7 @@ def add_referenced_license_detection_from_package(resource, codebase, no_license
         license_detection_mapping["identifier"] = get_new_identifier_from_detections(
             initial_detection=license_detection_mapping,
             detections_added=detections_added,
+            license_expression=license_expression,
         )
 
     if modified:

src/packagedcode/licensing.py

@@ -95,7 +95,7 @@ def _create_dir(location):
     from subprocess import CalledProcessError
 
     # this may fail with exceptions
-    cmd = 'git', 'describe',
+    cmd = 'git', 'describe', '--tags',
     try:
         output = check_output(cmd, stderr=STDOUT)
         __version__ = output.decode('utf-8').strip()

src/scancode_config.py

@@ -1,2 +1,2 @@
-path,type,declared_license_expression,scan_errors
-apache-2.0.LICENSE,file,apache-2.0 AND mit,
+path,type,declared_license_expression,declared_license_expression_spdx,scan_errors
+apache-2.0.LICENSE,file,apache-2.0 AND mit,Apache-2.0 AND MIT,

tests/formattedcode/data/csv/expressions/expected.csv

@@ -5,6 +5,8 @@
       "type": "file",
       "license_detections": [],
       "declared_license_expression": "apache-2.0 AND mit",
+      "declared_license_expression_spdx": "Apache-2.0 AND MIT",
+      "license_clues": [],
       "scan_errors": []
     }
   ]

tests/formattedcode/data/csv/expressions/scan.json

@@ -124,6 +124,8 @@ def test_can_process_live_scan_for_packages_with_root():
 
 
 def test_output_can_handle_non_ascii_paths():
+    # very small file with selective attributes just to test unicode, no need
+    # run scan again, update manually if required
     test_file = test_env.get_test_loc('csv/unicode.json')
     result_file = test_env.get_temp_file(extension='csv', file_name='test_csv')
     run_scan_click(['--from-json', test_file, '--csv', result_file])
@@ -250,6 +252,8 @@ def test_can_process_live_scan_for_packages_strip_root():
 
 @pytest.mark.scanslow
 def test_output_contains_license_expression():
+    # very small file with selective attributes just to test license_expressions,
+    # no need to run scan again, update manually if required
     test_file = test_env.get_test_loc('csv/expressions/scan.json')
     result_file = test_env.get_temp_file('csv')
     args = ['--from-json', test_file, '--csv', result_file]
@@ -260,6 +264,7 @@ def test_output_contains_license_expression():
 
 @pytest.mark.scanslow
 def test_output_handles_non_standard_data():
+    # non standard data from other tool, update manually if required
     test_file = test_env.get_test_loc('csv/non-standard/identified.json')
     result_file = test_env.get_temp_file('csv')
     args = ['--from-json', test_file, '--csv', result_file]

tests/formattedcode/test_output_csv.py

@@ -84,7 +84,7 @@
           "detection_log": [
             "unknown-reference-to-local-file"
           ],
-          "identifier": "1c807a43-2040-70af-75aa-c343d5f2b90c"
+          "identifier": "apache-2.0-1c807a43-2040-70af-75aa-c343d5f2b90c"
         }
       ],
       "license_clues": [],