Show license detection issues using --todo option

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

Author: AyanSinhaMahapatra

scanpipe/filters.py

@@ -857,6 +857,7 @@ class LicenseFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
         choices=[(EMPTY_VAR, "None")] + CodebaseResource.Compliance.choices,
     )
     is_license_clue = StrictBooleanFilter()
+    needs_review = StrictBooleanFilter()
 
     class Meta:
         model = DiscoveredLicense
@@ -868,6 +869,7 @@ class Meta:
             "license_expression_spdx",
             "compliance_alert",
             "is_license_clue",
+            "needs_review",
         ]
 
 

scanpipe/migrations/0074_discovered_license_models.py

@@ -102,6 +102,20 @@ class Migration(migrations.Migration):
                         help_text='True if this is not a proper license detection which should be considered in the license_expression for the parent resource/package. A license match is considered as a clue if it could be a possiblefalse positives or the matched rule is tagged as a clue explicitly.',
                     ),
                 ),
+                (
+                    "from_package",
+                    models.BooleanField(
+                        default=False,
+                        help_text='True if this was discovered in a extracted license statement and False if this was discovered in a file.',
+                    ),
+                ),
+                (
+                    "needs_review",
+                    models.BooleanField(
+                        default=False,
+                        help_text='True if this was license detection needs to be reviewed as there might be a license detection issue.',
+                    ),
+                ),
                 (
                     "project",
                     models.ForeignKey(
@@ -111,6 +125,15 @@ class Migration(migrations.Migration):
                         to="scanpipe.project",
                     ),
                 ),
+                (
+                    "review_comments",
+                    models.JSONField(
+                        blank=True,
+                        default=list,
+                        help_text='A list of review comments for license detection issues which needs review. These descriptive comments are based on ambigous detection types and could also offers helpful suggestions on how to review/report these detection issues.',
+                        verbose_name='Review Comments',
+                    ),
+                ),
             ],
             options={
                 "ordering": ["detection_count", "identifier"],
@@ -130,6 +153,18 @@ class Migration(migrations.Migration):
                         fields=["detection_count"],
                         name="scanpipe_di_detecti_d87ff1_idx",
                     ),
+                    models.Index(
+                        fields=['is_license_clue'],
+                        name='scanpipe_di_is_lice_f4922a_idx'
+                    ),
+                    models.Index(
+                        fields=['from_package'],
+                        name='scanpipe_di_from_pa_6485b2_idx'
+                    ),
+                    models.Index(
+                        fields=['needs_review'],
+                        name='scanpipe_di_needs_r_5cff82_idx'
+                    ),
                 ],
             },
             bases=(

scanpipe/models.py

@@ -4146,6 +4146,12 @@ class DiscoveredLicenseQuerySet(
     ComplianceAlertQuerySetMixin,
     ProjectRelatedQuerySet,
 ):
+    def needs_review(self):
+        return self.filter(needs_review=True)
+
+    def does_not_need_review(self):
+        return self.filter(needs_review=False)
+
     def order_by_count_and_expression(self):
         """Order by detection count and license expression (identifer) fields."""
         return self.order_by("-detection_count", "identifier")
@@ -4214,9 +4220,13 @@ class DiscoveredLicense(
 
     license_expression_field = "license_expression"
 
-    # If this license was discovered in a extracted license statement
-    # this is True, and False if this was discovered in a file.
-    from_package = None
+    from_package = models.BooleanField(
+        default=False,
+        help_text=_(
+            "True if this was discovered in a extracted license statement "
+            "and False if this was discovered in a file."
+        ),
+    )
 
     is_license_clue = models.BooleanField(
         default=False,
@@ -4246,6 +4256,26 @@ class DiscoveredLicense(
         ),
     )
 
+    needs_review = models.BooleanField(
+        default=False,
+        help_text=_(
+            "True if this was license detection needs to be reviewed "
+            "as there might be a license detection issue."
+        ),
+    )
+
+    review_comments = models.JSONField(
+        _("Review Comments"),
+        default=list,
+        blank=True,
+        help_text=_(
+            "A list of review comments for license detection issues which "
+            "needs review. These descriptive comments are based on ambigous "
+            "detection types and could also offers helpful suggestions on "
+            "how to review/report these detection issues."
+        ),
+    )
+
     objects = DiscoveredLicenseQuerySet.as_manager()
 
     class Meta:
@@ -4255,6 +4285,9 @@ class Meta:
             models.Index(fields=["license_expression"]),
             models.Index(fields=["license_expression_spdx"]),
             models.Index(fields=["detection_count"]),
+            models.Index(fields=["is_license_clue"]),
+            models.Index(fields=["from_package"]),
+            models.Index(fields=["needs_review"]),
         ]
         constraints = [
             models.UniqueConstraint(
@@ -4268,7 +4301,7 @@ def __str__(self):
         return self.identifier
 
     @classmethod
-    def create_from_data(cls, project, detection_data):
+    def create_from_data(cls, project, detection_data, from_package=False):
         """
         Create and returns a DiscoveredLicense for a `project` from the
         `detection_data`. If one of the values of the required fields is not
@@ -4302,7 +4335,9 @@ def create_from_data(cls, project, detection_data):
             if field_name in cls.model_fields() and value not in EMPTY_VALUES
         }
 
-        discovered_license = cls(project=project, **cleaned_data)
+        discovered_license = cls(
+            project=project, from_package=from_package, **cleaned_data
+        )
         # Using save_error=False to not capture potential errors at this level but
         # rather in the CodebaseResource.create_and_add_license_data method so
         # resource data can be injected in the ProjectMessage record.

scanpipe/pipelines/scan_single_package.py

@@ -61,11 +61,14 @@ def steps(cls):
         "info": True,
         "license": True,
         "license_text": True,
+        "license_diagnostics": True,
+        "license_text_diagnostics": True,
         "license_references": True,
         "package": True,
         "url": True,
         "classify": True,
         "summary": True,
+        "todo": True,
     }
 
     def get_package_input(self):

scanpipe/pipes/__init__.py

@@ -329,6 +329,7 @@ def update_or_create_license_detection(
     from_package=False,
     count_detection=True,
     is_license_clue=False,
+    check_todo=False,
 ):
     """
     Get, update or create a DiscoveredLicense object then return it.
@@ -355,8 +356,9 @@ def update_or_create_license_detection(
         license_detection.update_from_data(detection_data)
     else:
         license_detection = DiscoveredLicense.create_from_data(
-            project,
-            detection_data,
+            project=project,
+            detection_data=detection_data,
+            from_package=from_package,
         )
 
     if not license_detection:
@@ -377,7 +379,9 @@ def update_or_create_license_detection(
             count_detection=count_detection,
         )
 
-    license_detection.from_package = from_package
+    if check_todo:
+        scancode.check_license_detection_for_issues(license_detection)
+
     return license_detection
 
 
@@ -399,6 +403,23 @@ def _clean_license_detection_data(detection_data):
     return detection_data
 
 
+def update_license_detection_with_issue(project, todo_issue):
+    detection_data = todo_issue.get("detection")
+    if "identifier" not in detection_data:
+        return
+
+    detection_identifier = detection_data.get("identifier")
+    license_detection = project.discoveredlicenses.get_or_none(
+        identifier=detection_identifier,
+    )
+    if license_detection:
+        review_comments = todo_issue.get("review_comments").values()
+        license_detection.update(
+            needs_review=True,
+            review_comments=list(review_comments),
+        )
+
+
 def get_dependencies(project, dependency_data):
     """
     Given a `dependency_data` mapping, get a list of DiscoveredDependency objects

scanpipe/pipes/input.py

@@ -108,6 +108,7 @@ def load_inventory_from_toolkit_scan(project, input_location):
     scancode.create_discovered_dependencies(
         project, scanned_codebase, strip_datafile_path_root=True
     )
+    scancode.load_todo_issues(project, scanned_codebase)
 
 
 def load_inventory_from_scanpipe(project, scan_data, extra_data_prefix=None):

scanpipe/pipes/scancode.py

@@ -43,12 +43,16 @@
 from licensedcode.detection import FileRegion
 from licensedcode.detection import LicenseDetectionFromResult
 from licensedcode.detection import LicenseMatchFromResult
+from licensedcode.detection import UniqueDetection
+from licensedcode.detection import get_ambiguous_license_detections_by_type
 from packagedcode import get_package_handler
 from packagedcode import models as packagedcode_models
 from scancode import Scanner
 from scancode import api as scancode_api
 from scancode import cli as scancode_cli
 from scancode.cli import run_scan as scancode_run_scan
+from summarycode.todo import ReviewComments
+from summarycode.todo import get_review_comments
 
 from aboutcode.pipeline import LoopProgress
 from scanpipe import pipes
@@ -489,6 +493,7 @@ def collect_and_create_license_detections(project):
                 project=project,
                 detection_data=detection_data,
                 resource_path=resource.path,
+                check_todo=True,
             )
 
         for clue_data in resource.license_clues:
@@ -497,6 +502,7 @@ def collect_and_create_license_detections(project):
                 detection_data=clue_data,
                 resource_path=resource.path,
                 is_license_clue=True,
+                check_todo=True,
             )
 
     for resource in project.codebaseresources.has_package_data():
@@ -511,6 +517,7 @@ def collect_and_create_license_detections(project):
                     detection_data=detection,
                     resource_path=resource.path,
                     from_package=True,
+                    check_todo=True,
                 )
 
             for detection in package_data.other_license_detections:
@@ -519,6 +526,7 @@ def collect_and_create_license_detections(project):
                     detection_data=detection,
                     resource_path=resource.path,
                     from_package=True,
+                    check_todo=True,
                 )
 
 
@@ -1022,6 +1030,54 @@ def create_discovered_licenses(project, scanned_codebase):
             pipes.update_or_create_license_detection(project, detection_data)
 
 
+def load_todo_issues(project, scanned_codebase):
+    if hasattr(scanned_codebase.attributes, "todo"):
+        for todo_issue in scanned_codebase.attributes.todo:
+            pipes.update_license_detection_with_issue(project, todo_issue)
+
+    license_clues = project.discoveredlicenses.filter(
+        is_license_clue=True,
+    )
+    license_clues.update(
+        needs_review=True,
+        review_comments=[ReviewComments.LICENSE_CLUES.value],
+    )
+
+
+def check_license_detection_for_issues(discovered_license):
+    file_regions = [
+        FileRegion(
+            path=file_region.get("path"),
+            start_line=file_region.get("start_line"),
+            end_line=file_region.get("end_line"),
+        )
+        for file_region in discovered_license.file_regions
+    ]
+    matches = [
+        LicenseMatchFromResult.from_dict(license_match)
+        for license_match in discovered_license.matches
+    ]
+    unique_detection = UniqueDetection(
+        identifier=discovered_license.identifier,
+        license_expression=discovered_license.license_expression,
+        license_expression_spdx=discovered_license.license_expression_spdx,
+        detection_count=discovered_license.detection_count,
+        detection_log=discovered_license.detection_log,
+        matches=matches,
+        file_regions=file_regions,
+    )
+    detections_by_issue_type = get_ambiguous_license_detections_by_type(
+        unique_license_detections=[unique_detection],
+    )
+    if detections_by_issue_type:
+        issue_type = next(iter(detections_by_issue_type))
+        review_comments = get_review_comments(detection_log=[issue_type])
+        discovered_license.update(
+            needs_review=True,
+            review_comments=list(review_comments.values()),
+        )
+
+
 def set_codebase_resource_for_package(codebase_resource, discovered_package):
     """
     Assign the `discovered_package` to the `codebase_resource` and set its

scanpipe/templates/scanpipe/license_detection_list.html

@@ -24,6 +24,11 @@
             <td style="min-width: 300px;" title="{{ license_detection.identifier }}">
               {# CAUTION: Avoid relying on get_absolute_url to prevent unnecessary query triggers #}
               <a href="{% url 'license_detail' project.slug license_detection.identifier %}">{{ license_detection.identifier }}</a>
+              {% if license_detection.needs_review %}
+                <a href="{% url 'license_detail' project.slug license_detection.identifier %}#detection">
+                  <i class="fa-solid fa-magnifying-glass fa-sm has-text-danger" title="License Detection Issues"></i>
+                </a>
+              {% endif %}
               {% if license_detection.has_compliance_alert %}
                 <a href="{% url 'license_detail' project.slug license_detection.identifier %}#detection">
                   <i class="fa-solid fa-scale-balanced fa-sm has-text-danger" title="License Compliance Alerts"></i>
@@ -42,6 +47,9 @@
             <td>
               {{ license_detection.is_license_clue }}
             </td>
+            <td>
+              {{ license_detection.needs_review }}
+            </td>
             {% if display_compliance_alert %}
               <td>
                 <a href="?compliance_alert={{ license_detection.compliance_alert }}" class="is-black-link">

scanpipe/templates/scanpipe/panels/license_detections_summary.html

@@ -18,6 +18,12 @@
       <a class="panel-block is-align-items-flex-start break-word is-flex is-align-items-center" href="{{ project_licenses_url }}?is_license_clue=False" target="_blank">
         See all license detections
         <span class="tag is-rounded ml-1">{{ total_counts.all|intcomma }}</span>
+        {% if total_counts.needs_review %}
+        <span class="has-text-danger is-size-6 ml-2">
+          <i class="fa-solid fa-magnifying-glass"></i>
+          {{ total_counts.needs_review|intcomma }}
+        </span>
+        {% endif %}
         {% if total_counts.with_compliance_error %}
         <span class="has-text-danger is-size-6 ml-2">
           <i class="fa-solid fa-scale-balanced fa-sm"></i>
@@ -30,6 +36,12 @@
       <a class="panel-block is-align-items-flex-start break-word is-flex is-align-items-center" href="{{ project_licenses_url }}?is_license_clue=True" target="_blank">
         See all license clues
         <span class="tag is-rounded ml-1">{{ clue_counts.all|intcomma }}</span>
+        {% if clue_counts.needs_review %}
+        <span class="has-text-danger is-size-6 ml-2">
+          <i class="fa-solid fa-magnifying-glass"></i>
+          {{ clue_counts.needs_review|intcomma }}
+        </span>
+        {% endif %}
         {% if clue_counts.with_compliance_error %}
         <span class="has-text-danger is-size-6 ml-2">
           <i class="fa-solid fa-scale-balanced fa-sm"></i>