Skip to content

Commit a3a2a64

Browse files
AyanSinhaMahapatraAyanSinhaMahapatra
authored
Support SCTK License detection models (#1124)
* Add initial LicenseDetection Models and UI Signed-off-by: Ayan Sinha Mahapatra <[email protected]> * Support LicenseDetection models in load_inventory Signed-off-by: Ayan Sinha Mahapatra <[email protected]> * Add License Detections in project summary Signed-off-by: Ayan Sinha Mahapatra <[email protected]> * Support LicenseDetection creation in all pipelines Signed-off-by: Ayan Sinha Mahapatra <[email protected]> * Fix linter checks Signed-off-by: Ayan Sinha Mahapatra <[email protected]> * Reposition migrations after rebasing main Signed-off-by: Ayan Sinha Mahapatra <[email protected]> * Fix SCIO tests Signed-off-by: Ayan Sinha Mahapatra <[email protected]> * Improve detail views with links in UI This commit improves license, package and resource details views with detection tabs and links to each other. Signed-off-by: Ayan Sinha Mahapatra <[email protected]> * Improve LicenseDetections UI Signed-off-by: Ayan Sinha Mahapatra <[email protected]> * Improve license detection and summary related views Signed-off-by: Ayan Sinha Mahapatra <[email protected]> * Add license detection improvements Signed-off-by: Ayan Sinha Mahapatra <[email protected]> * Add license detections to Compliance alerts panel Signed-off-by: Ayan Sinha Mahapatra <[email protected]> * Add support for viewing license clues Reference: #1355 Signed-off-by: Ayan Sinha Mahapatra <[email protected]> * Show license detection issues using `--todo` option Signed-off-by: Ayan Sinha Mahapatra <[email protected]> --------- Signed-off-by: Ayan Sinha Mahapatra <[email protected]>
1 parent 9b8308a commit a3a2a64

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+2194
-354
lines changed

‎scancodeio/settings.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,7 @@
120120
"resource": 100,
121121
"package": 100,
122122
"dependency": 100,
123+
"license": 100,
123124
"relation": 100,
124125
},
125126
)

‎scanpipe/api/serializers.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
from scanpipe.models import CodebaseRelation
3232
from scanpipe.models import CodebaseResource
3333
from scanpipe.models import DiscoveredDependency
34+
from scanpipe.models import DiscoveredLicense
3435
from scanpipe.models import DiscoveredPackage
3536
from scanpipe.models import InputSource
3637
from scanpipe.models import Project
@@ -469,6 +470,20 @@ class Meta:
469470
]
470471

471472

473+
class DiscoveredLicenseSerializer(serializers.ModelSerializer):
474+
compliance_alert = serializers.CharField()
475+
476+
class Meta:
477+
model = DiscoveredLicense
478+
fields = [
479+
"detection_count",
480+
"identifier",
481+
"license_expression",
482+
"license_expression_spdx",
483+
"compliance_alert",
484+
]
485+
486+
472487
class CodebaseRelationSerializer(serializers.ModelSerializer):
473488
from_resource = serializers.ReadOnlyField(source="from_resource.path")
474489
to_resource = serializers.ReadOnlyField(source="to_resource.path")
@@ -527,6 +542,7 @@ def get_model_serializer(model_class):
527542
CodebaseResource: CodebaseResourceSerializer,
528543
DiscoveredPackage: DiscoveredPackageSerializer,
529544
DiscoveredDependency: DiscoveredDependencySerializer,
545+
DiscoveredLicense: DiscoveredLicenseSerializer,
530546
CodebaseRelation: CodebaseRelationSerializer,
531547
ProjectMessage: ProjectMessageSerializer,
532548
}.get(model_class, None)

‎scanpipe/filters.py‎

Lines changed: 59 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
from scanpipe.models import CodebaseRelation
4141
from scanpipe.models import CodebaseResource
4242
from scanpipe.models import DiscoveredDependency
43+
from scanpipe.models import DiscoveredLicense
4344
from scanpipe.models import DiscoveredPackage
4445
from scanpipe.models import Project
4546
from scanpipe.models import ProjectMessage
@@ -549,6 +550,7 @@ class ResourceFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
549550
"related_from__from_resource__path",
550551
],
551552
)
553+
552554
compliance_alert = django_filters.ChoiceFilter(
553555
choices=[(EMPTY_VAR, "None")] + CodebaseResource.Compliance.choices,
554556
)
@@ -608,8 +610,8 @@ class Meta:
608610

609611
def __init__(self, *args, **kwargs):
610612
super().__init__(*args, **kwargs)
611-
license_expression_filer = self.filters["detected_license_expression"]
612-
license_expression_filer.extra["widget"] = HasValueDropdownWidget()
613+
license_expression_filter = self.filters["detected_license_expression"]
614+
license_expression_filter.extra["widget"] = HasValueDropdownWidget()
613615

614616

615617
class IsVulnerable(django_filters.ChoiceFilter):
@@ -639,6 +641,19 @@ def filter(self, qs, value):
639641
return super().filter(qs, value)
640642

641643

644+
class DiscoveredLicenseSearchFilter(QuerySearchFilter):
645+
def filter(self, qs, value):
646+
if not value:
647+
return qs
648+
649+
search_fields = ["license_expression", "license_expression_spdx"]
650+
lookups = Q()
651+
for field_names in search_fields:
652+
lookups |= Q(**{f"{field_names}__{self.lookup_expr}": value})
653+
654+
return qs.filter(lookups)
655+
656+
642657
class GroupOrderingFilter(django_filters.OrderingFilter):
643658
"""Add the ability to provide a group a fields to order by."""
644659

@@ -816,6 +831,48 @@ class Meta:
816831
]
817832

818833

834+
class LicenseFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
835+
dropdown_widget_fields = [
836+
"compliance_alert",
837+
"license_expression",
838+
"license_expression_spdx",
839+
]
840+
841+
search = DiscoveredLicenseSearchFilter(
842+
label="Search", field_name="name", lookup_expr="icontains"
843+
)
844+
sort = GroupOrderingFilter(
845+
label="Sort",
846+
fields=[
847+
"detection_count",
848+
"identifier",
849+
"license_expression",
850+
"license_expression_spdx",
851+
"compliance_alert",
852+
],
853+
)
854+
license_expression = django_filters.AllValuesFilter()
855+
license_expression_spdx = django_filters.AllValuesFilter()
856+
compliance_alert = django_filters.ChoiceFilter(
857+
choices=[(EMPTY_VAR, "None")] + CodebaseResource.Compliance.choices,
858+
)
859+
is_license_clue = StrictBooleanFilter()
860+
needs_review = StrictBooleanFilter()
861+
862+
class Meta:
863+
model = DiscoveredLicense
864+
fields = [
865+
"search",
866+
"identifier",
867+
"detection_count",
868+
"license_expression",
869+
"license_expression_spdx",
870+
"compliance_alert",
871+
"is_license_clue",
872+
"needs_review",
873+
]
874+
875+
819876
class ProjectMessageFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
820877
search = QuerySearchFilter(
821878
label="Search", search_fields=["description"], lookup_expr="icontains"

‎scanpipe/migrations/0074_discovered_license_models.py‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
# Generated by Django 5.0.2 on 2024-03-17 12:38
2+
3+
import django.db.models.deletion
4+
import scanpipe.models
5+
from django.db import migrations, models
6+
7+
8+
class Migration(migrations.Migration):
9+
10+
dependencies = [
11+
("scanpipe", "0073_add_sha1_git_checksum"),
12+
]
13+
14+
operations = [
15+
migrations.CreateModel(
16+
name="DiscoveredLicense",
17+
fields=[
18+
(
19+
"id",
20+
models.AutoField(
21+
auto_created=True,
22+
primary_key=True,
23+
serialize=False,
24+
verbose_name="ID",
25+
),
26+
),
27+
(
28+
"compliance_alert",
29+
models.CharField(
30+
blank=True,
31+
choices=[
32+
("ok", "Ok"),
33+
("warning", "Warning"),
34+
("error", "Error"),
35+
("missing", "Missing"),
36+
],
37+
editable=False,
38+
help_text="Indicates how the license expression complies with provided policies.",
39+
max_length=10,
40+
),
41+
),
42+
(
43+
"license_expression",
44+
models.TextField(
45+
blank=True,
46+
help_text="A license expression string using the SPDX license expression syntax and ScanCode license keys, the effective license expression for this license detection.",
47+
),
48+
),
49+
(
50+
"license_expression_spdx",
51+
models.TextField(
52+
blank=True,
53+
help_text="SPDX license expression string with SPDX ids.",
54+
),
55+
),
56+
(
57+
"matches",
58+
models.JSONField(
59+
blank=True,
60+
default=list,
61+
help_text="List of license matches combined in this detection.",
62+
verbose_name="Reference Matches",
63+
),
64+
),
65+
(
66+
"detection_log",
67+
models.JSONField(
68+
blank=True,
69+
default=list,
70+
help_text="A list of detection DetectionRule explaining how this detection was created.",
71+
),
72+
),
73+
(
74+
"identifier",
75+
models.CharField(
76+
blank=True,
77+
help_text="An identifier unique for a license detection, containing the license expression and a UUID crafted from the match contents.",
78+
max_length=1024,
79+
),
80+
),
81+
(
82+
"detection_count",
83+
models.BigIntegerField(
84+
blank=True,
85+
help_text="Total number of this license detection discovered.",
86+
null=True,
87+
),
88+
),
89+
(
90+
"file_regions",
91+
models.JSONField(
92+
blank=True,
93+
default=list,
94+
help_text="A list of file regions with resource path, start and end line details for each place this license detection was discovered at. Also contains whether this license was discovered from a file or from package metadata.",
95+
verbose_name="Detection Locations",
96+
),
97+
),
98+
(
99+
"is_license_clue",
100+
models.BooleanField(
101+
default=False,
102+
help_text='True if this is not a proper license detection which should be considered in the license_expression for the parent resource/package. A license match is considered as a clue if it could be a possiblefalse positives or the matched rule is tagged as a clue explicitly.',
103+
),
104+
),
105+
(
106+
"from_package",
107+
models.BooleanField(
108+
default=False,
109+
help_text='True if this was discovered in a extracted license statement and False if this was discovered in a file.',
110+
),
111+
),
112+
(
113+
"needs_review",
114+
models.BooleanField(
115+
default=False,
116+
help_text='True if this was license detection needs to be reviewed as there might be a license detection issue.',
117+
),
118+
),
119+
(
120+
"project",
121+
models.ForeignKey(
122+
editable=False,
123+
on_delete=django.db.models.deletion.CASCADE,
124+
related_name="%(class)ss",
125+
to="scanpipe.project",
126+
),
127+
),
128+
(
129+
"review_comments",
130+
models.JSONField(
131+
blank=True,
132+
default=list,
133+
help_text='A list of review comments for license detection issues which needs review. These descriptive comments are based on ambigous detection types and could also offers helpful suggestions on how to review/report these detection issues.',
134+
verbose_name='Review Comments',
135+
),
136+
),
137+
],
138+
options={
139+
"ordering": ["detection_count", "identifier"],
140+
"indexes": [
141+
models.Index(
142+
fields=["identifier"], name="scanpipe_di_identif_b533f3_idx"
143+
),
144+
models.Index(
145+
fields=["license_expression"],
146+
name="scanpipe_di_license_33d11a_idx",
147+
),
148+
models.Index(
149+
fields=["license_expression_spdx"],
150+
name="scanpipe_di_license_eb5e9d_idx",
151+
),
152+
models.Index(
153+
fields=["detection_count"],
154+
name="scanpipe_di_detecti_d87ff1_idx",
155+
),
156+
models.Index(
157+
fields=['is_license_clue'],
158+
name='scanpipe_di_is_lice_f4922a_idx'
159+
),
160+
models.Index(
161+
fields=['from_package'],
162+
name='scanpipe_di_from_pa_6485b2_idx'
163+
),
164+
models.Index(
165+
fields=['needs_review'],
166+
name='scanpipe_di_needs_r_5cff82_idx'
167+
),
168+
],
169+
},
170+
bases=(
171+
scanpipe.models.UpdateMixin,
172+
scanpipe.models.SaveProjectMessageMixin,
173+
scanpipe.models.UpdateFromDataMixin,
174+
models.Model,
175+
),
176+
),
177+
migrations.AddConstraint(
178+
model_name="discoveredlicense",
179+
constraint=models.UniqueConstraint(
180+
condition=models.Q(("identifier", ""), _negated=True),
181+
fields=("project", "identifier"),
182+
name="scanpipe_discoveredlicense_unique_license_id_within_project",
183+
),
184+
),
185+
]

0 commit comments

Comments
 (0)