Add support for viewing license clues

Reference: #1355
Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

Author: AyanSinhaMahapatra

scanpipe/filters.py

@@ -856,6 +856,7 @@ class LicenseFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
     compliance_alert = django_filters.ChoiceFilter(
         choices=[(EMPTY_VAR, "None")] + CodebaseResource.Compliance.choices,
     )
+    is_license_clue = StrictBooleanFilter()
 
     class Meta:
         model = DiscoveredLicense
@@ -866,6 +867,7 @@ class Meta:
             "license_expression",
             "license_expression_spdx",
             "compliance_alert",
+            "is_license_clue",
         ]
 
 

scanpipe/migrations/0074_discovered_license_models.py

@@ -95,6 +95,13 @@ class Migration(migrations.Migration):
                         verbose_name="Detection Locations",
                     ),
                 ),
+                (
+                    "is_license_clue",
+                    models.BooleanField(
+                        default=False,
+                        help_text='True if this is not a proper license detection which should be considered in the license_expression for the parent resource/package. A license match is considered as a clue if it could be a possiblefalse positives or the matched rule is tagged as a clue explicitly.',
+                    ),
+                ),
                 (
                     "project",
                     models.ForeignKey(

scanpipe/models.py

@@ -2321,10 +2321,10 @@ def without_symlinks(self):
         return self.filter(~Q(type=self.model.Type.SYMLINK))
 
     def has_license_detections(self):
-        return self.filter(~Q(license_detections=[]))
+        return self.filter(~Q(license_detections=[]) | ~Q(license_clues=[]))
 
     def has_no_license_detections(self):
-        return self.filter(license_detections=[])
+        return self.filter(Q(license_detections=[]) & Q(license_clues=[]))
 
     def has_package_data(self):
         return self.filter(~Q(package_data=[]))
@@ -4218,6 +4218,16 @@ class DiscoveredLicense(
     # this is True, and False if this was discovered in a file.
     from_package = None
 
+    is_license_clue = models.BooleanField(
+        default=False,
+        help_text=_(
+            "True if this is not a proper license detection which should be "
+            "considered in the license_expression for the parent resource/package. "
+            "A license match is considered as a clue if it could be a possible"
+            "false positives or the matched rule is tagged as a clue explicitly."
+        ),
+    )
+
     detection_count = models.BigIntegerField(
         blank=True,
         null=True,

scanpipe/pipes/__init__.py

@@ -328,6 +328,7 @@ def update_or_create_license_detection(
     resource_path=None,
     from_package=False,
     count_detection=True,
+    is_license_clue=False,
 ):
     """
     Get, update or create a DiscoveredLicense object then return it.
@@ -339,7 +340,11 @@ def update_or_create_license_detection(
     already. `from_package` is True if the license detection was in a
     `extracted_license_statement` from a package metadata.
     """
+    if is_license_clue:
+        detection_data = scancode.get_detection_data_from_clue(detection_data)
+
     detection_identifier = detection_data["identifier"]
+    detection_data["is_license_clue"] = is_license_clue
 
     license_detection = project.discoveredlicenses.get_or_none(
         identifier=detection_identifier,
@@ -355,10 +360,10 @@ def update_or_create_license_detection(
         )
 
     if not license_detection:
+        detection_data["resource_path"] = resource_path
         project.add_error(
             model="update_or_create_license_detection",
             details=detection_data,
-            resource=resource_path,
         )
         return
 

scanpipe/pipes/scancode.py

@@ -39,7 +39,10 @@
 from commoncode import fileutils
 from commoncode.resource import VirtualCodebase
 from extractcode import api as extractcode_api
+from licensedcode.detection import DetectionCategory
 from licensedcode.detection import FileRegion
+from licensedcode.detection import LicenseDetectionFromResult
+from licensedcode.detection import LicenseMatchFromResult
 from packagedcode import get_package_handler
 from packagedcode import models as packagedcode_models
 from scancode import Scanner
@@ -488,6 +491,14 @@ def collect_and_create_license_detections(project):
                 resource_path=resource.path,
             )
 
+        for clue_data in resource.license_clues:
+            pipes.update_or_create_license_detection(
+                project=project,
+                detection_data=clue_data,
+                resource_path=resource.path,
+                is_license_clue=True,
+            )
+
     for resource in project.codebaseresources.has_package_data():
         for package_mapping in resource.package_data:
             package_data = packagedcode_models.PackageData.from_dict(
@@ -511,6 +522,28 @@ def collect_and_create_license_detections(project):
                 )
 
 
+def get_detection_data_from_clue(clue_data):
+    """
+    From a LicenseMatch mapping, create a LicenseDetection mapping by
+    populating the identifier and license_expression fields.
+    """
+    license_match = LicenseMatchFromResult.from_dict(clue_data)
+    license_detection = LicenseDetectionFromResult.from_matches(
+        matches=[license_match],
+        analysis=DetectionCategory.LICENSE_CLUES.value,
+    )
+    license_detection.license_expression = license_match.rule.license_expression
+    license_detection.license_expression_spdx = (
+        license_match.rule.spdx_license_expression()
+    )
+    license_detection.identifier = license_detection.identifier_with_expression
+    return license_detection.to_dict(
+        include_text=True,
+        license_diagnostics=True,
+        license_text_diagnostics=True,
+    )
+
+
 def get_file_region(detection_data, resource_path):
     """
     From a LicenseDetection mapping `detection_data`, create a FileRegion
@@ -909,6 +942,16 @@ def create_codebase_resources(project, scanned_codebase):
             )
             logger.debug(f"Add {codebase_resource} to {detection_identifier}")
 
+        license_clues = getattr(scanned_resource, "license_clues", [])
+        for clue_data in license_clues:
+            pipes.update_or_create_license_detection(
+                project=project,
+                detection_data=clue_data,
+                resource_path=resource_path,
+                is_license_clue=True,
+            )
+            logger.debug(f"Add license clue at {codebase_resource}")
+
         packages = getattr(scanned_resource, "package_data", [])
         for package_data in packages:
             license_detections = package_data.get("license_detections", [])

scanpipe/templates/scanpipe/license_detection_list.html

@@ -39,6 +39,9 @@
             <td>
               {{ license_detection.detection_count }}
             </td>
+            <td>
+              {{ license_detection.is_license_clue }}
+            </td>
             {% if display_compliance_alert %}
               <td>
                 <a href="?compliance_alert={{ license_detection.compliance_alert }}" class="is-black-link">

scanpipe/templates/scanpipe/panels/license_detections_summary.html

@@ -15,7 +15,7 @@
         </a>
       {% endfor %}
       {% if total_counts.all %}
-      <a class="panel-block is-align-items-flex-start break-word is-flex is-align-items-center" href="{{ project_licenses_url }}" target="_blank">
+      <a class="panel-block is-align-items-flex-start break-word is-flex is-align-items-center" href="{{ project_licenses_url }}?is_license_clue=False" target="_blank">
         See all license detections
         <span class="tag is-rounded ml-1">{{ total_counts.all|intcomma }}</span>
         {% if total_counts.with_compliance_error %}
@@ -26,6 +26,18 @@
         {% endif %}
       </a>
       {% endif %}
+      {% if license_clues %}
+      <a class="panel-block is-align-items-flex-start break-word is-flex is-align-items-center" href="{{ project_licenses_url }}?is_license_clue=True" target="_blank">
+        See all license clues
+        <span class="tag is-rounded ml-1">{{ clue_counts.all|intcomma }}</span>
+        {% if clue_counts.with_compliance_error %}
+        <span class="has-text-danger is-size-6 ml-2">
+          <i class="fa-solid fa-scale-balanced fa-sm"></i>
+          {{ clue_counts.with_compliance_error|intcomma }}
+        </span>
+        {% endif %}
+      </a>
+      {% endif %}
     </nav>
   </div>
 {% endif %}

scanpipe/templates/scanpipe/tabset/tab_resource_detections.html

@@ -30,7 +30,7 @@
         <thead>
           <tr>
             <th>License expression</th>
-            <th>License clue detials</th>
+            <th>License clue details</th>
           </tr>
         </thead>
         <tbody>

scanpipe/views.py

@@ -1107,8 +1107,11 @@ class ProjectLicenseDetectionSummaryView(ConditionalLoginRequired, generic.Detai
 
     @staticmethod
     def get_license_detection_summary(project, limit=10):
+        proper_license_detections = project.discoveredlicenses.filter(
+            is_license_clue=False,
+        )
         license_counter = count_group_by(
-            project.discoveredlicenses, "license_expression"
+            proper_license_detections, "license_expression"
         )
 
         if list(license_counter.keys()) == [""]:
@@ -1126,7 +1129,7 @@ def get_license_detection_summary(project, limit=10):
         expressions_with_compliance_alert = []
         for license_expression in top_licenses.keys():
             has_compliance_alert = (
-                project.discoveredlicenses.filter(license_expression=license_expression)
+                proper_license_detections.filter(license_expression=license_expression)
                 .has_compliance_alert()
                 .exists()
             )
@@ -1135,21 +1138,39 @@ def get_license_detection_summary(project, limit=10):
 
         total_counts = {
             "with_compliance_error": (
-                project.discoveredlicenses.has_compliance_alert().count()
+                proper_license_detections.has_compliance_alert().count()
             ),
-            "all": project.discoveredlicenses.count(),
+            "all": proper_license_detections.count(),
         }
 
-        return top_licenses, expressions_with_compliance_alert, total_counts
+        license_clues = project.discoveredlicenses.filter(
+            is_license_clue=True,
+        )
+        clue_counts = {}
+        if license_clues.exists():
+            clue_counts = {
+                "with_compliance_error": (license_clues.has_compliance_alert().count()),
+                "all": license_clues.count(),
+            }
+
+        return (
+            top_licenses,
+            expressions_with_compliance_alert,
+            total_counts,
+            license_clues,
+            clue_counts,
+        )
 
     def get_context_data(self, **kwargs):
         context = super().get_context_data(**kwargs)
-        summary, expressions, counts = self.get_license_detection_summary(
-            project=self.object
+        summary, expressions, counts, clues, clue_counts = (
+            self.get_license_detection_summary(project=self.object)
         )
         context["license_detection_summary"] = summary
         context["expressions_with_compliance_alert"] = expressions
         context["total_counts"] = counts
+        context["license_clues"] = clues
+        context["clue_counts"] = clue_counts
         context["project_licenses_url"] = reverse(
             "project_licenses", args=[self.object.slug]
         )
@@ -1850,6 +1871,7 @@ class DiscoveredLicenseListView(
             "filter_fieldname": "license_expression_spdx",
         },
         "detection_count",
+        "is_license_clue",
         {
             "field_name": "compliance_alert",
             "filter_fieldname": "compliance_alert",