Support SCTK License detection models

scancodeio/settings.py

@@ -120,6 +120,7 @@
         "resource": 100,
         "package": 100,
         "dependency": 100,
+        "license": 100,
         "relation": 100,
     },
 )

scanpipe/api/serializers.py

@@ -31,6 +31,7 @@
 from scanpipe.models import CodebaseRelation
 from scanpipe.models import CodebaseResource
 from scanpipe.models import DiscoveredDependency
+from scanpipe.models import DiscoveredLicense
 from scanpipe.models import DiscoveredPackage
 from scanpipe.models import InputSource
 from scanpipe.models import Project
@@ -469,6 +470,20 @@ class Meta:
         ]
 
 
+class DiscoveredLicenseSerializer(serializers.ModelSerializer):
+    compliance_alert = serializers.CharField()
+
+    class Meta:
+        model = DiscoveredLicense
+        fields = [
+            "detection_count",
+            "identifier",
+            "license_expression",
+            "license_expression_spdx",
+            "compliance_alert",
+        ]
+
+
 class CodebaseRelationSerializer(serializers.ModelSerializer):
     from_resource = serializers.ReadOnlyField(source="from_resource.path")
     to_resource = serializers.ReadOnlyField(source="to_resource.path")
@@ -527,6 +542,7 @@ def get_model_serializer(model_class):
         CodebaseResource: CodebaseResourceSerializer,
         DiscoveredPackage: DiscoveredPackageSerializer,
         DiscoveredDependency: DiscoveredDependencySerializer,
+        DiscoveredLicense: DiscoveredLicenseSerializer,
         CodebaseRelation: CodebaseRelationSerializer,
         ProjectMessage: ProjectMessageSerializer,
     }.get(model_class, None)

scanpipe/filters.py

@@ -40,6 +40,7 @@
 from scanpipe.models import CodebaseRelation
 from scanpipe.models import CodebaseResource
 from scanpipe.models import DiscoveredDependency
+from scanpipe.models import DiscoveredLicense
 from scanpipe.models import DiscoveredPackage
 from scanpipe.models import Project
 from scanpipe.models import ProjectMessage
@@ -549,6 +550,7 @@ class ResourceFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
             "related_from__from_resource__path",
         ],
     )
+
     compliance_alert = django_filters.ChoiceFilter(
         choices=[(EMPTY_VAR, "None")] + CodebaseResource.Compliance.choices,
     )
@@ -608,8 +610,8 @@ class Meta:
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        license_expression_filer = self.filters["detected_license_expression"]
-        license_expression_filer.extra["widget"] = HasValueDropdownWidget()
+        license_expression_filter = self.filters["detected_license_expression"]
+        license_expression_filter.extra["widget"] = HasValueDropdownWidget()
 
 
 class IsVulnerable(django_filters.ChoiceFilter):
@@ -639,6 +641,19 @@ def filter(self, qs, value):
         return super().filter(qs, value)
 
 
+class DiscoveredLicenseSearchFilter(QuerySearchFilter):
+    def filter(self, qs, value):
+        if not value:
+            return qs
+
+        search_fields = ["license_expression", "license_expression_spdx"]
+        lookups = Q()
+        for field_names in search_fields:
+            lookups |= Q(**{f"{field_names}__{self.lookup_expr}": value})
+
+        return qs.filter(lookups)
+
+
 class GroupOrderingFilter(django_filters.OrderingFilter):
     """Add the ability to provide a group a fields to order by."""
 
@@ -816,6 +831,48 @@ class Meta:
         ]
 
 
+class LicenseFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
+    dropdown_widget_fields = [
+        "compliance_alert",
+        "license_expression",
+        "license_expression_spdx",
+    ]
+
+    search = DiscoveredLicenseSearchFilter(
+        label="Search", field_name="name", lookup_expr="icontains"
+    )
+    sort = GroupOrderingFilter(
+        label="Sort",
+        fields=[
+            "detection_count",
+            "identifier",
+            "license_expression",
+            "license_expression_spdx",
+            "compliance_alert",
+        ],
+    )
+    license_expression = django_filters.AllValuesFilter()
+    license_expression_spdx = django_filters.AllValuesFilter()
+    compliance_alert = django_filters.ChoiceFilter(
+        choices=[(EMPTY_VAR, "None")] + CodebaseResource.Compliance.choices,
+    )
+    is_license_clue = StrictBooleanFilter()
+    needs_review = StrictBooleanFilter()
+
+    class Meta:
+        model = DiscoveredLicense
+        fields = [
+            "search",
+            "identifier",
+            "detection_count",
+            "license_expression",
+            "license_expression_spdx",
+            "compliance_alert",
+            "is_license_clue",
+            "needs_review",
+        ]
+
+
 class ProjectMessageFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
     search = QuerySearchFilter(
         label="Search", search_fields=["description"], lookup_expr="icontains"

scanpipe/migrations/0074_discovered_license_models.py

@@ -0,0 +1,185 @@
+# Generated by Django 5.0.2 on 2024-03-17 12:38
+
+import django.db.models.deletion
+import scanpipe.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("scanpipe", "0073_add_sha1_git_checksum"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="DiscoveredLicense",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "compliance_alert",
+                    models.CharField(
+                        blank=True,
+                        choices=[
+                            ("ok", "Ok"),
+                            ("warning", "Warning"),
+                            ("error", "Error"),
+                            ("missing", "Missing"),
+                        ],
+                        editable=False,
+                        help_text="Indicates how the license expression complies with provided policies.",
+                        max_length=10,
+                    ),
+                ),
+                (
+                    "license_expression",
+                    models.TextField(
+                        blank=True,
+                        help_text="A license expression string using the SPDX license expression syntax and ScanCode license keys, the effective license expression for this license detection.",
+                    ),
+                ),
+                (
+                    "license_expression_spdx",
+                    models.TextField(
+                        blank=True,
+                        help_text="SPDX license expression string with SPDX ids.",
+                    ),
+                ),
+                (
+                    "matches",
+                    models.JSONField(
+                        blank=True,
+                        default=list,
+                        help_text="List of license matches combined in this detection.",
+                        verbose_name="Reference Matches",
+                    ),
+                ),
+                (
+                    "detection_log",
+                    models.JSONField(
+                        blank=True,
+                        default=list,
+                        help_text="A list of detection DetectionRule explaining how this detection was created.",
+                    ),
+                ),
+                (
+                    "identifier",
+                    models.CharField(
+                        blank=True,
+                        help_text="An identifier unique for a license detection, containing the license expression and a UUID crafted from the match contents.",
+                        max_length=1024,
+                    ),
+                ),
+                (
+                    "detection_count",
+                    models.BigIntegerField(
+                        blank=True,
+                        help_text="Total number of this license detection discovered.",
+                        null=True,
+                    ),
+                ),
+                (
+                    "file_regions",
+                    models.JSONField(
+                        blank=True,
+                        default=list,
+                        help_text="A list of file regions with resource path, start and end line details for each place this license detection was discovered at. Also contains whether this license was discovered from a file or from package metadata.",
+                        verbose_name="Detection Locations",
+                    ),
+                ),
+                (
+                    "is_license_clue",
+                    models.BooleanField(
+                        default=False,
+                        help_text='True if this is not a proper license detection which should be considered in the license_expression for the parent resource/package. A license match is considered as a clue if it could be a possiblefalse positives or the matched rule is tagged as a clue explicitly.',
+                    ),
+                ),
+                (
+                    "from_package",
+                    models.BooleanField(
+                        default=False,
+                        help_text='True if this was discovered in a extracted license statement and False if this was discovered in a file.',
+                    ),
+                ),
+                (
+                    "needs_review",
+                    models.BooleanField(
+                        default=False,
+                        help_text='True if this was license detection needs to be reviewed as there might be a license detection issue.',
+                    ),
+                ),
+                (
+                    "project",
+                    models.ForeignKey(
+                        editable=False,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="%(class)ss",
+                        to="scanpipe.project",
+                    ),
+                ),
+                (
+                    "review_comments",
+                    models.JSONField(
+                        blank=True,
+                        default=list,
+                        help_text='A list of review comments for license detection issues which needs review. These descriptive comments are based on ambigous detection types and could also offers helpful suggestions on how to review/report these detection issues.',
+                        verbose_name='Review Comments',
+                    ),
+                ),
+            ],
+            options={
+                "ordering": ["detection_count", "identifier"],
+                "indexes": [
+                    models.Index(
+                        fields=["identifier"], name="scanpipe_di_identif_b533f3_idx"
+                    ),
+                    models.Index(
+                        fields=["license_expression"],
+                        name="scanpipe_di_license_33d11a_idx",
+                    ),
+                    models.Index(
+                        fields=["license_expression_spdx"],
+                        name="scanpipe_di_license_eb5e9d_idx",
+                    ),
+                    models.Index(
+                        fields=["detection_count"],
+                        name="scanpipe_di_detecti_d87ff1_idx",
+                    ),
+                    models.Index(
+                        fields=['is_license_clue'],
+                        name='scanpipe_di_is_lice_f4922a_idx'
+                    ),
+                    models.Index(
+                        fields=['from_package'],
+                        name='scanpipe_di_from_pa_6485b2_idx'
+                    ),
+                    models.Index(
+                        fields=['needs_review'],
+                        name='scanpipe_di_needs_r_5cff82_idx'
+                    ),
+                ],
+            },
+            bases=(
+                scanpipe.models.UpdateMixin,
+                scanpipe.models.SaveProjectMessageMixin,
+                scanpipe.models.UpdateFromDataMixin,
+                models.Model,
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="discoveredlicense",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(("identifier", ""), _negated=True),
+                fields=("project", "identifier"),
+                name="scanpipe_discoveredlicense_unique_license_id_within_project",
+            ),
+        ),
+    ]