Add identifier to resource/package license detections

* Top level license detections are lean, without matches
* Resource/package level license detections have the identifier
* identifier is computed scan time, and recomputed in post-processing
  if detections are combined.

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

Author: AyanSinhaMahapatra

src/licensedcode/detection.py

@@ -174,6 +174,14 @@ class LicenseDetection:
         )
     )
 
+    identifier = attr.ib(
+        default=None,
+        metadata=dict(
+            help='An identifier unique for a license detection, containing the license '
+            'expression and a UUID crafted from the match contents.')
+    )
+
+
     # Only used in unique detection calculation and referencing
     file_region = attr.ib(
         default=attr.Factory(dict),
@@ -221,11 +229,13 @@ def from_matches(
                 detection_log=detection_log,
             )
 
-        return cls(
+        detection = cls(
             matches=matches,
             license_expression=str(license_expression),
             detection_log=detection_log,
         )
+        detection.identifier = detection.identifier_with_expression
+        return detection
 
     def __eq__(self, other):
         return (
@@ -258,25 +268,23 @@ def get_file_region(self, path):
         )
 
     @property
-    def identifier(self):
+    def _identifier(self):
         """
         Return an unique identifier for a license detection, based on it's
         underlying license matches with the tokenized matched_text.
         """
         data = []
         for match in self.matches:
-            tokenized_matched_text = tuple(query_tokenizer(match.matched_text))
+            tokenized_matched_text = tuple(query_tokenizer(match.matched_text()))
             identifier = (
-                match.identifier,
+                match.rule.identifier,
                 match.score(),
                 tokenized_matched_text,
             )
             data.append(identifier)
 
         # Return a uuid generated from the contents of the matches
-        identifier_string = repr(tuple(data))
-        md_hash = sha1(identifier_string.encode('utf-8'))
-        return str(uuid.UUID(hex=md_hash.hexdigest()[:32]))
+        return get_uuid_on_content(content=data)
 
     @property
     def identifier_with_expression(self):
@@ -285,7 +293,7 @@ def identifier_with_expression(self):
         and an UUID created from the detection contents.
         """
         id_safe_expression = python_safe_name(s=str(self.license_expression))
-        return "{}-{}".format(id_safe_expression, self.identifier)
+        return "{}-{}".format(id_safe_expression, self._identifier)
 
     def get_start_end_line(self):
         """
@@ -448,6 +456,16 @@ def dict_fields(attr, value):
         return detection
 
 
+def get_uuid_on_content(content):
+    """
+    Return an UUID based on the contents of a list, which should be
+    a list of hashable elements.
+    """
+    identifier_string = repr(tuple(content))
+    md_hash = sha1(identifier_string.encode('utf-8'))
+    return str(uuid.UUID(hex=md_hash.hexdigest()[:32]))
+
+
 @attr.s
 class LicenseDetectionFromResult(LicenseDetection):
     """
@@ -475,6 +493,7 @@ def from_license_detection_mapping(
         detection = cls(
             license_expression=license_detection_mapping["license_expression"],
             detection_log=license_detection_mapping.get("detection_log", []) or None,
+            identifier=license_detection_mapping["identifier"],
             matches=matches,
             file_region=None,
         )
@@ -503,6 +522,20 @@ def detections_from_license_detection_mappings(
     return license_detections
 
 
+def get_new_identifier_from_detections(initial_detection, detections_added):
+    """
+    Return a new UUID based on two sets of detections: `initial_detection` is
+    the detection being modified with a list of detections (from another file region)
+    `detections_added`.
+    """
+    identifiers = [
+        detection_mapping["identifier"]
+        for detection_mapping in detections_added
+    ]
+    identifiers.append(initial_detection["identifier"])
+    return get_uuid_on_content(content=sorted(identifiers))
+
+
 @attr.s
 class LicenseMatchFromResult(LicenseMatch):
     """
@@ -611,7 +644,7 @@ def get_unique_detections(cls, license_detections):
             detection_mapping = detection.to_dict()
             unique_license_detections.append(
                 cls(
-                    identifier=detection.identifier_with_expression,
+                    identifier=detection_mapping["identifier"],
                     license_expression=detection_mapping["license_expression"],
                     detection_log=detection_mapping.get("detection_log", []) or [],
                     matches=detection_mapping["matches"],
@@ -629,6 +662,9 @@ def dict_fields(attr, value):
             if attr.name == 'files':
                 return False
 
+            if attr.name == 'matches':
+                return False
+
             if attr.name == 'detection_log' and not license_diagnostics:
                 return False
 
@@ -1361,6 +1397,7 @@ def process_detections(detections, licensing=Licensing()):
                         licensing=licensing,
                     ))
                     detection.detection_log.append(DetectionRule.NOT_LICENSE_CLUES.value)
+                    detection.identifier = detection.identifier_with_expression
 
             yield detection
 

src/licensedcode/plugin_license.py

@@ -22,6 +22,7 @@
 from licensedcode.detection import find_referenced_resource
 from licensedcode.detection import get_detected_license_expression
 from licensedcode.detection import get_matches_from_detection_mappings
+from licensedcode.detection import get_new_identifier_from_detections
 from licensedcode.detection import get_referenced_filenames
 from licensedcode.detection import group_matches
 from licensedcode.detection import process_detections
@@ -64,7 +65,6 @@ class LicenseScanner(ScanPlugin):
         ('license_detections', attr.ib(default=attr.Factory(list))),
         ('license_clues', attr.ib(default=attr.Factory(list))),
         ('percentage_of_license_text', attr.ib(default=0)),
-        ('for_license_detections', attr.ib(default=attr.Factory(list))),
     ])
 
     codebase_attributes = dict(
@@ -189,7 +189,10 @@ def process_codebase(self, codebase, license_diagnostics, **kwargs):
         if codebase.has_single_resource and not codebase.root.is_file:
             return
 
-        license_detections = collect_license_detections(codebase)
+        license_detections = collect_license_detections(
+            codebase=codebase,
+            include_license_clues=False
+        )
         unique_license_detections = UniqueDetection.get_unique_detections(
             license_detections=license_detections,
         )
@@ -218,35 +221,13 @@ def process_codebase(self, codebase, license_diagnostics, **kwargs):
                     f'after : {license_expressions_after}'
                 )
 
-        populate_for_license_detections_in_resources(
-            codebase=codebase,
-            detections=unique_license_detections,
-        )
         codebase.attributes.license_detections.extend([
             unique_detection.to_dict(license_diagnostics=license_diagnostics)
             for unique_detection in unique_license_detections
         ])
 
 
-def populate_for_license_detections_in_resources(codebase, detections):
-    """
-    Populate the ``codebase`` resource ``for_license_detections`` attribute from
-    a ``detections`` list of ``UniqueDetection``,
-    """
-    for detection in detections:
-        if TRACE:
-            logger_debug(
-                f'populate_for_license_detections_in_resources:',
-                f'for detection: {detection.license_expression}\n',
-                f'file paths: {detection.files}',
-            )
-        for file_region in detection.files:
-            resource = codebase.get_resource(path=file_region.path)
-            resource.for_license_detections.append(detection.identifier)
-            codebase.save_resource(resource)
-
-
-def collect_license_detections(codebase):
+def collect_license_detections(codebase, include_license_clues=True):
     """
     Return a list of LicenseDetectionFromResult from a ``codebase``
     """
@@ -269,7 +250,7 @@ def collect_license_detections(codebase):
                 )
                 resource_license_detections.extend(license_detection_objects)
 
-            if license_clues:
+            if include_license_clues and license_clues:
                 license_matches = LicenseMatchFromResult.from_dicts(
                     license_match_mappings=license_clues,
                 )
@@ -336,6 +317,7 @@ def add_referenced_filenames_license_matches_for_detections(resource, codebase):
             file_path=resource.path,
         )
         detection_modified = False
+        detections_added = []
         license_match_mappings = license_detection_mapping["matches"]
         referenced_filenames = get_referenced_filenames(license_detection.matches)
 
@@ -352,6 +334,7 @@ def add_referenced_filenames_license_matches_for_detections(resource, codebase):
             if referenced_resource and referenced_resource.license_detections:
                 modified = True
                 detection_modified = True
+                detections_added.extend(referenced_resource.license_detections)
                 license_match_mappings.extend(
                     get_matches_from_detection_mappings(
                         license_detections=referenced_resource.license_detections
@@ -368,6 +351,10 @@ def add_referenced_filenames_license_matches_for_detections(resource, codebase):
         )
         license_detection_mapping["license_expression"] = str(license_expression)
         license_detection_mapping["detection_log"] = detection_log
+        license_detection_mapping["identifier"] = get_new_identifier_from_detections(
+            initial_detection=license_detection_mapping,
+            detections_added=detections_added,
+        )
 
     if modified:
         license_expressions = [

src/packagedcode/licensing.py

@@ -19,6 +19,7 @@
 from licensedcode.detection import group_matches
 from licensedcode.detection import get_detected_license_expression
 from licensedcode.detection import get_matches_from_detection_mappings
+from licensedcode.detection import get_new_identifier_from_detections
 from licensedcode.detection import get_unknown_license_detection
 from licensedcode.detection import get_referenced_filenames
 from licensedcode.detection import find_referenced_resource
@@ -132,6 +133,10 @@ def add_referenced_license_matches_for_package(resource, codebase, no_licenses):
             )
             license_detection_mapping["license_expression"] = str(license_expression)
             license_detection_mapping["detection_log"] = detection_log
+            license_detection_mapping["identifier"] = get_new_identifier_from_detections(
+                initial_detection=license_detection_mapping,
+                detections_added=referenced_license_detections,
+            )
 
         if modified:
             license_expressions = [
@@ -184,6 +189,7 @@ def add_referenced_license_detection_from_package(resource, codebase, no_license
         )
         detection_modified = False
         license_match_mappings = license_detection_mapping["matches"]
+        detections_added = []
         referenced_filenames = get_referenced_filenames(license_matches=license_detection_object.matches)
         if not referenced_filenames:
             continue
@@ -214,6 +220,7 @@ def add_referenced_license_detection_from_package(resource, codebase, no_license
                 modified = True
                 detection_modified = True
                 license_match_mappings.extend(sibling_detection["matches"])
+                detections_added.append(sibling_detection)
                 analysis = DetectionCategory.UNKNOWN_REFERENCE_IN_FILE_TO_NONEXISTENT_PACKAGE.value
 
         else:
@@ -229,6 +236,7 @@ def add_referenced_license_detection_from_package(resource, codebase, no_license
                     modified = True
                     detection_modified = True
                     license_match_mappings.extend(pkg_detection["matches"])
+                    detections_added.append(pkg_detection)
                     analysis = DetectionCategory.UNKNOWN_REFERENCE_IN_FILE_TO_PACKAGE.value
 
         if not detection_modified:
@@ -241,6 +249,10 @@ def add_referenced_license_detection_from_package(resource, codebase, no_license
         )
         license_detection_mapping["license_expression"] = str(license_expression)
         license_detection_mapping["detection_log"] = detection_log
+        license_detection_mapping["identifier"] = get_new_identifier_from_detections(
+            initial_detection=license_detection_mapping,
+            detections_added=detections_added,
+        )
 
     if modified: