From 82f7ce6112620ba43e9047c4edf81650fa4f19ad Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 20 Mar 2024 17:20:59 +0530 Subject: [PATCH 01/14] Add initial LicenseDetection Models and UI Signed-off-by: Ayan Sinha Mahapatra --- scancodeio/settings.py | 1 + scanpipe/api/serializers.py | 16 ++ scanpipe/filters.py | 49 ++++++ .../0054_discovered_license_models.py | 141 +++++++++++++++++ scanpipe/models.py | 145 ++++++++++++++++++ .../scanpipe/license_detection_detail.html | 13 ++ .../scanpipe/license_detection_list.html | 59 +++++++ scanpipe/urls.py | 10 ++ scanpipe/views.py | 74 +++++++++ 9 files changed, 508 insertions(+) create mode 100644 scanpipe/migrations/0054_discovered_license_models.py create mode 100644 scanpipe/templates/scanpipe/license_detection_detail.html create mode 100644 scanpipe/templates/scanpipe/license_detection_list.html diff --git a/scancodeio/settings.py b/scancodeio/settings.py index 112de2deff..bab8f3ca02 100644 --- a/scancodeio/settings.py +++ b/scancodeio/settings.py @@ -108,6 +108,7 @@ "resource": 100, "package": 100, "dependency": 100, + "license": 100, "relation": 100, }, ) diff --git a/scanpipe/api/serializers.py b/scanpipe/api/serializers.py index 5f41732465..f1e9ad88c5 100644 --- a/scanpipe/api/serializers.py +++ b/scanpipe/api/serializers.py @@ -31,6 +31,7 @@ from scanpipe.models import CodebaseResource from scanpipe.models import DiscoveredDependency from scanpipe.models import DiscoveredPackage +from scanpipe.models import DiscoveredLicense from scanpipe.models import InputSource from scanpipe.models import Project from scanpipe.models import ProjectMessage @@ -413,6 +414,20 @@ class Meta: ] +class DiscoveredLicenseSerializer(serializers.ModelSerializer): + compliance_alert = serializers.CharField() + + class Meta: + model = DiscoveredLicense + fields = [ + "detection_count", + "identifier", + "license_expression", + "license_expression_spdx", + "compliance_alert", + ] + + class CodebaseRelationSerializer(serializers.ModelSerializer): from_resource = serializers.ReadOnlyField(source="from_resource.path") to_resource = serializers.ReadOnlyField(source="to_resource.path") @@ -471,6 +486,7 @@ def get_model_serializer(model_class): CodebaseResource: CodebaseResourceSerializer, DiscoveredPackage: DiscoveredPackageSerializer, DiscoveredDependency: DiscoveredDependencySerializer, + DiscoveredLicense: DiscoveredLicenseSerializer, CodebaseRelation: CodebaseRelationSerializer, ProjectMessage: ProjectMessageSerializer, }.get(model_class, None) diff --git a/scanpipe/filters.py b/scanpipe/filters.py index cdba64d8aa..92ef8e6353 100644 --- a/scanpipe/filters.py +++ b/scanpipe/filters.py @@ -39,6 +39,7 @@ from scanpipe.models import CodebaseResource from scanpipe.models import DiscoveredDependency from scanpipe.models import DiscoveredPackage +from scanpipe.models import DiscoveredLicense from scanpipe.models import Project from scanpipe.models import ProjectMessage from scanpipe.models import Run @@ -591,6 +592,19 @@ def filter(self, qs, value): return qs.filter(lookups) +class DiscoveredLicenseSearchFilter(QuerySearchFilter): + def filter(self, qs, value): + if not value: + return qs + + search_fields = ["license_expression", "license_expression_spdx"] + lookups = Q() + for field_names in search_fields: + lookups |= Q(**{f"{field_names}__{self.lookup_expr}": value}) + + return qs.filter(lookups) + + class GroupOrderingFilter(django_filters.OrderingFilter): """Add the ability to provide a group a fields to order by.""" @@ -755,6 +769,41 @@ class Meta: ] +class LicenseFilterSet(FilterSetUtilsMixin, django_filters.FilterSet): + dropdown_widget_fields = [ + "compliance_alert", + ] + + search = DiscoveredLicenseSearchFilter( + label="Search", field_name="name", lookup_expr="icontains" + ) + sort = GroupOrderingFilter( + label="Sort", + fields=[ + "detection_count", + "identifier", + "license_expression", + "license_expression_spdx", + "compliance_alert", + ], + ) + license_expression = ParentAllValuesFilter() + compliance_alert = django_filters.ChoiceFilter( + choices=[(EMPTY_VAR, "None")] + CodebaseResource.Compliance.choices, + ) + + class Meta: + model = DiscoveredLicense + fields = [ + "search", + "identifier", + "detection_count", + "license_expression", + "license_expression_spdx", + "compliance_alert", + ] + + class ProjectMessageFilterSet(FilterSetUtilsMixin, django_filters.FilterSet): search = QuerySearchFilter( label="Search", field_name="description", lookup_expr="icontains" diff --git a/scanpipe/migrations/0054_discovered_license_models.py b/scanpipe/migrations/0054_discovered_license_models.py new file mode 100644 index 0000000000..d4253eb6f8 --- /dev/null +++ b/scanpipe/migrations/0054_discovered_license_models.py @@ -0,0 +1,141 @@ +# Generated by Django 5.0.2 on 2024-03-17 12:38 + +import django.db.models.deletion +import scanpipe.models +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("scanpipe", "0053_restructure_pipelines_data"), + ] + + operations = [ + migrations.CreateModel( + name="DiscoveredLicense", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "compliance_alert", + models.CharField( + blank=True, + choices=[ + ("ok", "Ok"), + ("warning", "Warning"), + ("error", "Error"), + ("missing", "Missing"), + ], + editable=False, + help_text="Indicates how the license expression complies with provided policies.", + max_length=10, + ), + ), + ( + "license_expression", + models.TextField( + blank=True, + help_text="A license expression string using the SPDX license expression syntax and ScanCode license keys, the effective license expression for this license detection.", + ), + ), + ( + "license_expression_spdx", + models.TextField( + blank=True, + help_text="SPDX license expression string with SPDX ids.", + ), + ), + ( + "matches", + models.JSONField( + blank=True, + default=list, + help_text="List of license matches combined in this detection.", + ), + ), + ( + "detection_log", + models.JSONField( + blank=True, + default=list, + help_text="A list of detection DetectionRule explaining how this detection was created.", + ), + ), + ( + "identifier", + models.CharField( + blank=True, + help_text="An identifier unique for a license detection, containing the license expression and a UUID crafted from the match contents.", + max_length=1024, + ), + ), + ( + "detection_count", + models.BigIntegerField( + blank=True, + help_text="Total number of this license detection discovered.", + null=True, + ), + ), + ( + "file_regions", + models.JSONField( + blank=True, + default=list, + help_text="A list of file regions with resource path, start and end line details for each place this license detection was discovered at. Also contains whether this license was discovered from a file or from package metadata.", + ), + ), + ( + "project", + models.ForeignKey( + editable=False, + on_delete=django.db.models.deletion.CASCADE, + related_name="%(class)ss", + to="scanpipe.project", + ), + ), + ], + options={ + "ordering": ["detection_count", "identifier"], + "indexes": [ + models.Index( + fields=["identifier"], name="scanpipe_di_identif_b533f3_idx" + ), + models.Index( + fields=["license_expression"], + name="scanpipe_di_license_33d11a_idx", + ), + models.Index( + fields=["license_expression_spdx"], + name="scanpipe_di_license_eb5e9d_idx", + ), + models.Index( + fields=["detection_count"], + name="scanpipe_di_detecti_d87ff1_idx", + ), + ], + }, + bases=( + scanpipe.models.UpdateMixin, + scanpipe.models.SaveProjectMessageMixin, + scanpipe.models.UpdateFromDataMixin, + models.Model, + ), + ), + migrations.AddConstraint( + model_name="discoveredlicense", + constraint=models.UniqueConstraint( + condition=models.Q(("identifier", ""), _negated=True), + fields=("project", "identifier"), + name="scanpipe_discoveredlicense_unique_license_id_within_project", + ), + ), + ] diff --git a/scanpipe/models.py b/scanpipe/models.py index 412fb985d6..b0f9c9d41f 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -612,6 +612,7 @@ def delete_related_objects(self): self.projectmessages, self.codebaserelations, self.discovereddependencies, + self.discoveredlicenses, self.codebaseresources, self.runs, ] @@ -3473,6 +3474,150 @@ def as_spdx(self): ) +class DiscoveredLicenseQuerySet(ProjectRelatedQuerySet): + def order_by_count_and_expression(self): + """Order by detection count and license expression (identifer) fields.""" + return self.order_by("detection_count", "identifier") + + +class AbstractLicenseDetection(models.Model): + """ + These fields should be kept in line with + `licensedcode.detection.LicenseDetection`. + """ + + license_expression = models.TextField( + blank=True, + help_text=_( + 'A license expression string using the SPDX license expression' + ' syntax and ScanCode license keys, the effective license expression' + ' for this license detection.' + ), + ) + + license_expression_spdx = models.TextField( + blank=True, + help_text=_('SPDX license expression string with SPDX ids.'), + ) + + matches = models.JSONField( + default=list, + blank=True, + help_text=_('List of license matches combined in this detection.'), + ) + + detection_log = models.JSONField( + default=list, + blank=True, + help_text=_( + 'A list of detection DetectionRule explaining how ' + 'this detection was created.' + ), + ) + + identifier = models.CharField( + max_length=1024, + blank=True, + help_text=_( + 'An identifier unique for a license detection, containing the license ' + 'expression and a UUID crafted from the match contents.' + ), + ) + + class Meta: + abstract = True + + +class DiscoveredLicense( + ProjectRelatedModel, + SaveProjectMessageMixin, + UpdateFromDataMixin, + ComplianceAlertMixin, + AbstractLicenseDetection, +): + """ + A project's Discovered Licenses are the unique License Detection objects + discovered in the code under analysis. + + """ + license_expression_field = "license_expression" + + detection_count = models.BigIntegerField( + blank=True, + null=True, + help_text=_("Total number of this license detection discovered."), + ) + + file_regions = models.JSONField( + default=list, + blank=True, + help_text=_( + 'A list of file regions with resource path, start and end line ' + 'details for each place this license detection was discovered at. ' + 'Also contains whether this license was discovered from a file or ' + 'from package metadata.' + ), + ) + + objects = DiscoveredLicenseQuerySet.as_manager() + + class Meta: + ordering = ["detection_count", "identifier"] + indexes = [ + models.Index(fields=["identifier"]), + models.Index(fields=["license_expression"]), + models.Index(fields=["license_expression_spdx"]), + models.Index(fields=["detection_count"]), + ] + constraints = [ + models.UniqueConstraint( + fields=["project", "identifier"], + condition=~Q(identifier=""), + name="%(app_label)s_%(class)s_unique_license_id_within_project", + ), + ] + + def __str__(self): + return self.identifier + + @classmethod + def create_from_data(cls, project, detection_data): + """ + Create and returns a DiscoveredLicense for a `project` from the `detection_data`. + If one of the values of the required fields is not available, a "ProjectMessage" + is created instead of a new DiscoveredLicense instance. + """ + detection_data = detection_data.copy() + required_fields = ["license_expression", "identifier", "matches"] + missing_values = [ + field_name + for field_name in required_fields + if not detection_data.get(field_name) + ] + + if missing_values: + message = ( + f"No values for the following required fields: " + f"{', '.join(missing_values)}" + ) + + project.add_warning(description=message, model=cls, details=detection_data) + return + + cleaned_data = { + field_name: value + for field_name, value in detection_data.items() + if field_name in cls.model_fields() and value not in EMPTY_VALUES + } + + discovered_license = cls(project=project, **cleaned_data) + # Using save_error=False to not capture potential errors at this level but + # rather in the CodebaseResource.create_and_add_license_data method so resource data + # can be injected in the ProjectMessage record. + discovered_license.save(save_error=False, capture_exception=False) + return discovered_license + + class WebhookSubscription(UUIDPKModel, ProjectRelatedModel): target_url = models.URLField(_("Target URL"), max_length=1024) created_date = models.DateTimeField(auto_now_add=True, editable=False) diff --git a/scanpipe/templates/scanpipe/license_detection_detail.html b/scanpipe/templates/scanpipe/license_detection_detail.html new file mode 100644 index 0000000000..eec34ad59d --- /dev/null +++ b/scanpipe/templates/scanpipe/license_detection_detail.html @@ -0,0 +1,13 @@ +{% extends "scanpipe/base.html" %} +{% load static %} + +{% block title %}ScanCode.io: {{ project.name }} - {{ object.name }}{% endblock %} + +{% block content %} +
+ {% include 'scanpipe/includes/navbar_header.html' %} +
{% include 'scanpipe/includes/messages.html' %}
+ {% include 'scanpipe/includes/breadcrumb_detail_view.html' with object_title=object.identifier url_name="project_licenses" %} + {% include 'scanpipe/tabset/tabset.html' %} +
+{% endblock %} \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/license_detection_list.html b/scanpipe/templates/scanpipe/license_detection_list.html new file mode 100644 index 0000000000..426c5b1452 --- /dev/null +++ b/scanpipe/templates/scanpipe/license_detection_list.html @@ -0,0 +1,59 @@ +{% extends "scanpipe/base.html" %} + +{% block title %}ScanCode.io: {{ project.name }} - License Detections{% endblock %} + +{% block content %} +
+ {% include 'scanpipe/includes/navbar_header.html' %} +
+
+ {% include 'scanpipe/includes/breadcrumb.html' with linked_project=True current="License Detections" %} + {% include 'scanpipe/includes/search_field.html' with extra_class="is-small" %} +
+ {% include 'scanpipe/includes/pagination_header.html' %} + {% include 'scanpipe/includes/filters_breadcrumb.html' with filterset=filter only %} +
+
+ +
+ + {% include 'scanpipe/includes/list_view_thead.html' %} + + {% for license_detection in object_list %} + + + + + + {% if display_compliance_alert %} + + {% endif %} + + {% empty %} + + + + {% endfor %} + +
+ {# CAUTION: Avoid relying on get_absolute_url to prevent unnecessary query triggers #} + {{ license_detection.identifier }} + + {{ license_detection.license_expression }} + + {{ license_detection.license_expression_spdx }} + + {{ license_detection.detection_count }} + + + {{ package.compliance_alert }} + +
+ No Licenses detected. Clear search and filters +
+ + {% if is_paginated %} + {% include 'scanpipe/includes/pagination.html' with page_obj=page_obj %} + {% endif %} +
+{% endblock %} \ No newline at end of file diff --git a/scanpipe/urls.py b/scanpipe/urls.py index 3391c00c4e..5469c7e1a0 100644 --- a/scanpipe/urls.py +++ b/scanpipe/urls.py @@ -51,6 +51,11 @@ views.DiscoveredPackageDetailsView.as_view(), name="package_detail", ), + path( + "project//license_detections//", + views.DiscoveredLicenseDetailsView.as_view(), + name="license_detail", + ), path( "project//dependencies//", views.DiscoveredDependencyDetailsView.as_view(), @@ -61,6 +66,11 @@ views.DiscoveredPackageListView.as_view(), name="project_packages", ), + path( + "project//license_detections/", + views.DiscoveredLicenseListView.as_view(), + name="project_licenses", + ), path( "project//dependencies/", views.DiscoveredDependencyListView.as_view(), diff --git a/scanpipe/views.py b/scanpipe/views.py index 22dd676fe6..ec0afe5fb9 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -65,6 +65,7 @@ from scanpipe.filters import PAGE_VAR from scanpipe.filters import DependencyFilterSet from scanpipe.filters import PackageFilterSet +from scanpipe.filters import LicenseFilterSet from scanpipe.filters import ProjectFilterSet from scanpipe.filters import ProjectMessageFilterSet from scanpipe.filters import RelationFilterSet @@ -82,6 +83,7 @@ from scanpipe.models import CodebaseResource from scanpipe.models import DiscoveredDependency from scanpipe.models import DiscoveredPackage +from scanpipe.models import DiscoveredLicense from scanpipe.models import InputSource from scanpipe.models import Project from scanpipe.models import ProjectMessage @@ -1517,6 +1519,46 @@ def get_queryset(self): return super().get_queryset().order_by("dependency_uid") +class DiscoveredLicenseListView( + ConditionalLoginRequired, + ProjectRelatedViewMixin, + TableColumnsMixin, + ExportXLSXMixin, + PaginatedFilterView, +): + model = DiscoveredLicense + filterset_class = LicenseFilterSet + template_name = "scanpipe/license_detection_list.html" + paginate_by = settings.SCANCODEIO_PAGINATE_BY.get("license", 10) + table_columns = [ + "identifier", + { + "field_name": "license_expression", + "filter_fieldname": "license_expression", + }, + "license_expression_spdx", + "detection_count", + { + "field_name": "compliance_alert", + "condition": scanpipe_app.policies_enabled, + "filter_fieldname": "compliance_alert", + }, + ] + + def get_queryset(self): + return ( + super() + .get_queryset() + .only( + "detection_count", + "license_expression", + "license_expression_spdx", + "compliance_alert", + ) + .order_by_count_and_expression() + ) + + class ProjectMessageListView( ConditionalLoginRequired, ProjectRelatedViewMixin, @@ -1967,6 +2009,38 @@ def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context["dependency_data"] = DiscoveredDependencySerializer(self.object).data return context + + +class DiscoveredLicenseDetailsView( + ConditionalLoginRequired, + ProjectRelatedViewMixin, + TabSetMixin, + generic.DetailView, +): + model = DiscoveredLicense + model_label = "license_detections" + slug_field = "identifier" + slug_url_kwarg = "identifier" + template_name = "scanpipe/license_detection_detail.html" + tabset = { + "essentials": { + "fields": [ + "license_expression", + "license_expression_spdx", + "identifier", + "detection_count", + ], + "icon_class": "fa-solid fa-info-circle", + }, + "detection": { + "fields": [ + {"field_name": "matches", "render_func": render_as_yaml}, + {"field_name": "detection_log", "render_func": render_as_yaml}, + {"field_name": "file_regions", "render_func": render_as_yaml}, + ], + "icon_class": "fa-solid fa-search", + }, + } @conditional_login_required From 7536957d1ccfe032760d9a981c9e8160c1b5baa8 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 20 Mar 2024 17:21:32 +0530 Subject: [PATCH 02/14] Support LicenseDetection models in load_inventory Signed-off-by: Ayan Sinha Mahapatra --- scanpipe/pipes/__init__.py | 32 ++++++++++++++++++++++++++++++++ scanpipe/pipes/input.py | 15 +++++++++++---- scanpipe/pipes/scancode.py | 19 +++++++++++++++++++ 3 files changed, 62 insertions(+), 4 deletions(-) diff --git a/scanpipe/pipes/__init__.py b/scanpipe/pipes/__init__.py index 7e8c737f61..be7c7aff8b 100644 --- a/scanpipe/pipes/__init__.py +++ b/scanpipe/pipes/__init__.py @@ -37,6 +37,7 @@ from scanpipe.models import CodebaseResource from scanpipe.models import DiscoveredDependency from scanpipe.models import DiscoveredPackage +from scanpipe.models import DiscoveredLicense from scanpipe.pipes import scancode logger = logging.getLogger("scanpipe.pipes") @@ -245,6 +246,37 @@ def update_or_create_dependency( return dependency +def update_or_create_license_detection(project, detection_data): + """ + Get, update or create a DiscoveredLicense object then return it. + Use the `project` and `detection_data` mapping to lookup and creates the + DiscoveredLicense using its detection identifier as a unique key. + """ + detection_identifier = detection_data["identifier"] + + license_detection = project.discoveredlicenses.get_or_none( + identifier=detection_identifier, + ) + detection_data = _clean_license_detection_data(detection_data) + + if license_detection: + license_detection.update_from_data(detection_data) + else: + license_detection = DiscoveredLicense.create_from_data( + project, + detection_data, + ) + + return license_detection + + +def _clean_license_detection_data(detection_data): + detection_data = detection_data.copy() + matches = detection_data.pop("sample_matches") + detection_data["matches"] = matches + return detection_data + + def get_or_create_relation(project, relation_data): """ Get or create a CodebaseRelation then return it. diff --git a/scanpipe/pipes/input.py b/scanpipe/pipes/input.py index 56befe4511..41e5eb16af 100644 --- a/scanpipe/pipes/input.py +++ b/scanpipe/pipes/input.py @@ -35,6 +35,7 @@ from scanpipe.models import CodebaseResource from scanpipe.models import DiscoveredDependency from scanpipe.models import DiscoveredPackage +from scanpipe.models import DiscoveredLicense from scanpipe.pipes import scancode from scanpipe.pipes.output import mappings_key_by_fieldname @@ -78,10 +79,11 @@ def is_archive(location): def load_inventory_from_toolkit_scan(project, input_location): """ - Create packages, dependencies, and resources loaded from the ScanCode-toolkit scan - results located at ``input_location``. + Create license detections, packages, dependencies, and resources + loaded from the ScanCode-toolkit scan results located at ``input_location``. """ scanned_codebase = scancode.get_virtual_codebase(project, input_location) + scancode.create_discovered_licenses(project, scanned_codebase) scancode.create_discovered_packages(project, scanned_codebase) scancode.create_codebase_resources(project, scanned_codebase) scancode.create_discovered_dependencies( @@ -91,9 +93,12 @@ def load_inventory_from_toolkit_scan(project, input_location): def load_inventory_from_scanpipe(project, scan_data): """ - Create packages, dependencies, resources, and relations loaded from a ScanCode.io - JSON output provided as ``scan_data``. + Create license detections, packages, dependencies, resources, and relations + loaded from a ScanCode.io JSON output provided as ``scan_data``. """ + for detection_data in scan_data.get("license_detections", []): + pipes.update_or_create_license_detection(project, detection_data) + for package_data in scan_data.get("packages", []): pipes.update_or_create_package(project, package_data) @@ -110,12 +115,14 @@ def load_inventory_from_scanpipe(project, scan_data): model_to_object_maker_func = { DiscoveredPackage: pipes.update_or_create_package, DiscoveredDependency: pipes.update_or_create_dependency, + DiscoveredLicense: pipes.update_or_create_license_detection, CodebaseResource: pipes.update_or_create_resource, CodebaseRelation: pipes.get_or_create_relation, } worksheet_name_to_model = { "PACKAGES": DiscoveredPackage, + "LICENSE_DETECTIONS": DiscoveredLicense, "RESOURCES": CodebaseResource, "DEPENDENCIES": DiscoveredDependency, "RELATIONS": CodebaseRelation, diff --git a/scanpipe/pipes/scancode.py b/scanpipe/pipes/scancode.py index 538d68d4ea..a1174a2210 100644 --- a/scanpipe/pipes/scancode.py +++ b/scanpipe/pipes/scancode.py @@ -586,6 +586,14 @@ def create_codebase_resources(project, scanned_codebase): discovered_package=package, ) + license_detections = getattr(scanned_resource, "license_detections", []) + for detection_data in license_detections: + detection_identifier = detection_data.get("identifier") + license_detection = project.discoveredlicenses.get_or_none( + identifier=detection_identifier + ) + logger.debug(f"Add {codebase_resource} to {license_detection}") + def create_discovered_packages(project, scanned_codebase): """ @@ -620,6 +628,17 @@ def create_discovered_dependencies( ) +def create_discovered_licenses(project, scanned_codebase): + """ + Save the license detections of a ScanCode `scanned_codebase` + scancode.resource.Codebase object to the database as a DiscoveredLicense of + `project`. + """ + if hasattr(scanned_codebase.attributes, "license_detections"): + for detection_data in scanned_codebase.attributes.license_detections: + pipes.update_or_create_license_detection(project, detection_data) + + def set_codebase_resource_for_package(codebase_resource, discovered_package): """ Assign the `discovered_package` to the `codebase_resource` and set its From 1a32a895ab113ed5165e424ea55584211480beed Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 21 Mar 2024 15:38:45 +0530 Subject: [PATCH 03/14] Add License Detections in project summary Signed-off-by: Ayan Sinha Mahapatra --- scanpipe/models.py | 7 ++++++- .../scanpipe/includes/project_summary_level.html | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/scanpipe/models.py b/scanpipe/models.py index b0f9c9d41f..44d72fb46d 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -1229,6 +1229,11 @@ def dependency_count(self): """Return the number of dependencies related to this project.""" return self.discovereddependencies.count() + @cached_property + def license_detections_count(self): + """Return the number of license detections in this project.""" + return self.discoveredlicenses.count() + @cached_property def message_count(self): """Return the number of messages related to this project.""" @@ -3477,7 +3482,7 @@ def as_spdx(self): class DiscoveredLicenseQuerySet(ProjectRelatedQuerySet): def order_by_count_and_expression(self): """Order by detection count and license expression (identifer) fields.""" - return self.order_by("detection_count", "identifier") + return self.order_by("-detection_count", "identifier") class AbstractLicenseDetection(models.Model): diff --git a/scanpipe/templates/scanpipe/includes/project_summary_level.html b/scanpipe/templates/scanpipe/includes/project_summary_level.html index 2dad4e5863..2e46585a87 100644 --- a/scanpipe/templates/scanpipe/includes/project_summary_level.html +++ b/scanpipe/templates/scanpipe/includes/project_summary_level.html @@ -40,6 +40,20 @@

+
+
+

License Detections

+

+ {% if project.license_detections_count %} + + {{ project.license_detections_count|intcomma }} + + {% else %} + 0 + {% endif %} +

+
+

Resources

From d1b43c134935a05659e976634f3534c0095a1791 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Mon, 25 Mar 2024 21:13:06 +0530 Subject: [PATCH 04/14] Support LicenseDetection creation in all pipelines Signed-off-by: Ayan Sinha Mahapatra --- scanpipe/models.py | 30 ++++++++++++ scanpipe/pipelines/deploy_to_develop.py | 8 ++++ scanpipe/pipelines/docker.py | 1 + scanpipe/pipelines/docker_windows.py | 1 + scanpipe/pipelines/root_filesystem.py | 8 ++++ scanpipe/pipelines/scan_codebase.py | 8 ++++ scanpipe/pipes/__init__.py | 22 +++++++-- scanpipe/pipes/scancode.py | 62 +++++++++++++++++++++++++ 8 files changed, 137 insertions(+), 3 deletions(-) diff --git a/scanpipe/models.py b/scanpipe/models.py index 44d72fb46d..c3ce73416a 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -2726,6 +2726,16 @@ def with_resources_count(self): ) return self.annotate(resources_count=count_subquery) + def has_license_detections(self): + return self.filter( + ~Q(license_detections=[]) | ~Q(other_license_detections=[]) + ) + + def has_no_license_detections(self): + return self.filter( + Q(license_detections=[]) & Q(other_license_detections=[]) + ) + class AbstractPackage(models.Model): """These fields should be kept in line with `packagedcode.models.PackageData`.""" @@ -3506,6 +3516,7 @@ class AbstractLicenseDetection(models.Model): ) matches = models.JSONField( + _("Reference Matches"), default=list, blank=True, help_text=_('List of license matches combined in this detection.'), @@ -3547,6 +3558,10 @@ class DiscoveredLicense( """ license_expression_field = "license_expression" + # If this license was discovered in a extracted license statement + # this is True, and False if this was discovered in a file. + from_package = None + detection_count = models.BigIntegerField( blank=True, null=True, @@ -3554,6 +3569,7 @@ class DiscoveredLicense( ) file_regions = models.JSONField( + _("Detection Locations"), default=list, blank=True, help_text=_( @@ -3622,6 +3638,20 @@ def create_from_data(cls, project, detection_data): discovered_license.save(save_error=False, capture_exception=False) return discovered_license + def update_with_file_region(self, file_region): + """ + If the `file_region` is a new file region, include it in the + `file_regions` list and increase the `detection_count` by 1. + """ + file_region_data = file_region.to_dict() + if not file_region_data in self.file_regions: + self.file_regions.append(file_region_data) + if not self.detection_count: + self.detection_count = 1 + else: + self.detection_count += 1 + self.save(update_fields=["detection_count", "file_regions"]) + class WebhookSubscription(UUIDPKModel, ProjectRelatedModel): target_url = models.URLField(_("Target URL"), max_length=1024) diff --git a/scanpipe/pipelines/deploy_to_develop.py b/scanpipe/pipelines/deploy_to_develop.py index 311495e220..f8add21b66 100644 --- a/scanpipe/pipelines/deploy_to_develop.py +++ b/scanpipe/pipelines/deploy_to_develop.py @@ -83,6 +83,7 @@ def steps(cls): cls.remove_packages_without_resources, cls.scan_unmapped_to_files, cls.scan_mapped_from_for_files, + cls.collect_and_create_license_detections, cls.flag_deployed_from_resources_with_missing_license, cls.create_local_files_packages, ) @@ -286,6 +287,13 @@ def scan_mapped_from_for_files(self): scan_files = d2d.get_from_files_for_scanning(self.project.codebaseresources) scancode.scan_for_files(self.project, scan_files, progress_logger=self.log) + def collect_and_create_license_detections(self): + """ + Collect and create unique license detections from resources and + package data. + """ + scancode.collect_and_create_license_detections(project=self.project) + def create_local_files_packages(self): """Create local-files packages for codebase resources not part of a package.""" d2d.create_local_files_packages(self.project) diff --git a/scanpipe/pipelines/docker.py b/scanpipe/pipelines/docker.py index 3e0897226a..c0ab10662a 100644 --- a/scanpipe/pipelines/docker.py +++ b/scanpipe/pipelines/docker.py @@ -42,6 +42,7 @@ def steps(cls): cls.flag_ignored_resources, cls.scan_for_application_packages, cls.scan_for_files, + cls.collect_and_create_license_detections, cls.analyze_scanned_files, cls.flag_not_analyzed_codebase_resources, ) diff --git a/scanpipe/pipelines/docker_windows.py b/scanpipe/pipelines/docker_windows.py index 2698824fba..3f7513131e 100644 --- a/scanpipe/pipelines/docker_windows.py +++ b/scanpipe/pipelines/docker_windows.py @@ -45,6 +45,7 @@ def steps(cls): cls.flag_ignored_resources, cls.scan_for_application_packages, cls.scan_for_files, + cls.collect_and_create_license_detections, cls.analyze_scanned_files, cls.flag_data_files_with_no_clues, cls.flag_not_analyzed_codebase_resources, diff --git a/scanpipe/pipelines/root_filesystem.py b/scanpipe/pipelines/root_filesystem.py index ebfae326ec..a40190268f 100644 --- a/scanpipe/pipelines/root_filesystem.py +++ b/scanpipe/pipelines/root_filesystem.py @@ -45,6 +45,7 @@ def steps(cls): cls.scan_for_application_packages, cls.match_not_analyzed_to_system_packages, cls.scan_for_files, + cls.collect_and_create_license_detections, cls.analyze_scanned_files, cls.flag_not_analyzed_codebase_resources, ) @@ -123,6 +124,13 @@ def scan_for_files(self): """Scan unknown resources for copyrights, licenses, emails, and urls.""" scancode.scan_for_files(self.project, progress_logger=self.log) + def collect_and_create_license_detections(self): + """ + Collect and create unique license detections from resources and + package data. + """ + scancode.collect_and_create_license_detections(project=self.project) + def analyze_scanned_files(self): """Analyze single file scan results for completeness.""" flag.analyze_scanned_files(self.project) diff --git a/scanpipe/pipelines/scan_codebase.py b/scanpipe/pipelines/scan_codebase.py index 668e71d6cd..45605fd213 100644 --- a/scanpipe/pipelines/scan_codebase.py +++ b/scanpipe/pipelines/scan_codebase.py @@ -45,6 +45,7 @@ def steps(cls): cls.flag_ignored_resources, cls.scan_for_application_packages, cls.scan_for_files, + cls.collect_and_create_license_detections, ) def copy_inputs_to_codebase_directory(self): @@ -65,3 +66,10 @@ def scan_for_application_packages(self): def scan_for_files(self): """Scan unknown resources for copyrights, licenses, emails, and urls.""" scancode.scan_for_files(self.project, progress_logger=self.log) + + def collect_and_create_license_detections(self): + """ + Collect and create unique license detections from resources and + package data. + """ + scancode.collect_and_create_license_detections(project=self.project) diff --git a/scanpipe/pipes/__init__.py b/scanpipe/pipes/__init__.py index be7c7aff8b..3a1bea41e5 100644 --- a/scanpipe/pipes/__init__.py +++ b/scanpipe/pipes/__init__.py @@ -246,11 +246,18 @@ def update_or_create_dependency( return dependency -def update_or_create_license_detection(project, detection_data): +def update_or_create_license_detection( + project, detection_data, resource_path, from_package=False, +): """ Get, update or create a DiscoveredLicense object then return it. Use the `project` and `detection_data` mapping to lookup and creates the DiscoveredLicense using its detection identifier as a unique key. + + Additonally if `resource_path` is passed, add the file region where + the license was detected to the DiscoveredLicense object, if not present + already. `from_package` is True if the license detection was in a + `extracted_license_statement` from a package metadata. """ detection_identifier = detection_data["identifier"] @@ -267,13 +274,22 @@ def update_or_create_license_detection(project, detection_data): detection_data, ) + if resource_path: + file_region = scancode.get_file_region( + detection_data=detection_data, + resource_path=resource_path, + ) + license_detection.update_with_file_region(file_region) + + license_detection.from_package = from_package return license_detection def _clean_license_detection_data(detection_data): detection_data = detection_data.copy() - matches = detection_data.pop("sample_matches") - detection_data["matches"] = matches + if "sample_matches" in detection_data: + matches = detection_data.pop("sample_matches") + detection_data["matches"] = matches return detection_data diff --git a/scanpipe/pipes/scancode.py b/scanpipe/pipes/scancode.py index a1174a2210..aadc4a62cd 100644 --- a/scanpipe/pipes/scancode.py +++ b/scanpipe/pipes/scancode.py @@ -39,6 +39,7 @@ from extractcode import api as extractcode_api from packagedcode import get_package_handler from packagedcode import models as packagedcode_models +from licensedcode.detection import FileRegion from scancode import Scanner from scancode import api as scancode_api from scancode import cli as scancode_cli @@ -411,6 +412,67 @@ def add_resource_to_package(package_uid, resource, project): resource.discovered_packages.add(package) +def collect_and_create_license_detections(project): + """ + Create instances of DiscoveredLicense for `project` from the parsed + license detections present in the CodebaseResources and + DiscoveredPackages of `project`. + """ + logger.info(f"Project {project} collect_license_detections:") + + for resource in project.codebaseresources.has_license_detections(): + logger.info(f" Processing: {resource.path} for licenses") + + for detection_data in resource.license_detections: + pipes.update_or_create_license_detection( + project=project, + detection_data=detection_data, + resource_path=resource.path, + ) + + for resource in project.codebaseresources.has_package_data(): + + for package_mapping in resource.package_data: + package_data = packagedcode_models.PackageData.from_dict( + mapping=package_mapping, + ) + + for detection in package_data.license_detections: + pipes.update_or_create_license_detection( + project=project, + detection_data=detection, + resource_path=resource.path, + from_package=True, + ) + + for detection in package_data.other_license_detections: + pipes.update_or_create_license_detection( + project=project, + detection_data=detection, + resource_path=resource.path, + from_package=True, + ) + + +def get_file_region(detection_data, resource_path): + """ + From a LicenseDetection mapping `detection_data`, create a FileRegion + object containing information about where this license was detected + exactly in a codebase, with `resource_path`, with start and end lines. + """ + start_line = min( + [match['start_line'] for match in detection_data["matches"]] + ) + end_line = max( + [match['end_line'] for match in detection_data["matches"]] + ) + return FileRegion( + path=resource_path, + start_line=start_line, + end_line=end_line, + ) + + def assemble_packages(project): """ Create instances of DiscoveredPackage and DiscoveredDependency for `project` From 96268abf4e05bf80ab79fba4b4ad741200ea2d26 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Mon, 25 Mar 2024 21:13:32 +0530 Subject: [PATCH 05/14] Fix linter checks Signed-off-by: Ayan Sinha Mahapatra --- scanpipe/api/serializers.py | 2 +- scanpipe/filters.py | 2 +- scanpipe/models.py | 56 +++++++++++++++++++------------------ scanpipe/pipes/__init__.py | 7 +++-- scanpipe/pipes/input.py | 2 +- scanpipe/pipes/scancode.py | 12 +++----- scanpipe/views.py | 6 ++-- 7 files changed, 44 insertions(+), 43 deletions(-) diff --git a/scanpipe/api/serializers.py b/scanpipe/api/serializers.py index f1e9ad88c5..4de2c4b09d 100644 --- a/scanpipe/api/serializers.py +++ b/scanpipe/api/serializers.py @@ -30,8 +30,8 @@ from scanpipe.models import CodebaseRelation from scanpipe.models import CodebaseResource from scanpipe.models import DiscoveredDependency -from scanpipe.models import DiscoveredPackage from scanpipe.models import DiscoveredLicense +from scanpipe.models import DiscoveredPackage from scanpipe.models import InputSource from scanpipe.models import Project from scanpipe.models import ProjectMessage diff --git a/scanpipe/filters.py b/scanpipe/filters.py index 92ef8e6353..57eada085b 100644 --- a/scanpipe/filters.py +++ b/scanpipe/filters.py @@ -38,8 +38,8 @@ from scanpipe.models import CodebaseRelation from scanpipe.models import CodebaseResource from scanpipe.models import DiscoveredDependency -from scanpipe.models import DiscoveredPackage from scanpipe.models import DiscoveredLicense +from scanpipe.models import DiscoveredPackage from scanpipe.models import Project from scanpipe.models import ProjectMessage from scanpipe.models import Run diff --git a/scanpipe/models.py b/scanpipe/models.py index c3ce73416a..52759901d3 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -2727,14 +2727,10 @@ def with_resources_count(self): return self.annotate(resources_count=count_subquery) def has_license_detections(self): - return self.filter( - ~Q(license_detections=[]) | ~Q(other_license_detections=[]) - ) + return self.filter(~Q(license_detections=[]) | ~Q(other_license_detections=[])) def has_no_license_detections(self): - return self.filter( - Q(license_detections=[]) & Q(other_license_detections=[]) - ) + return self.filter(Q(license_detections=[]) & Q(other_license_detections=[])) class AbstractPackage(models.Model): @@ -3504,30 +3500,30 @@ class AbstractLicenseDetection(models.Model): license_expression = models.TextField( blank=True, help_text=_( - 'A license expression string using the SPDX license expression' - ' syntax and ScanCode license keys, the effective license expression' - ' for this license detection.' + "A license expression string using the SPDX license expression" + " syntax and ScanCode license keys, the effective license expression" + " for this license detection." ), ) license_expression_spdx = models.TextField( blank=True, - help_text=_('SPDX license expression string with SPDX ids.'), + help_text=_("SPDX license expression string with SPDX ids."), ) matches = models.JSONField( _("Reference Matches"), default=list, blank=True, - help_text=_('List of license matches combined in this detection.'), + help_text=_("List of license matches combined in this detection."), ) detection_log = models.JSONField( default=list, blank=True, help_text=_( - 'A list of detection DetectionRule explaining how ' - 'this detection was created.' + "A list of detection DetectionRule explaining how " + "this detection was created." ), ) @@ -3535,8 +3531,8 @@ class AbstractLicenseDetection(models.Model): max_length=1024, blank=True, help_text=_( - 'An identifier unique for a license detection, containing the license ' - 'expression and a UUID crafted from the match contents.' + "An identifier unique for a license detection, containing the license " + "expression and a UUID crafted from the match contents." ), ) @@ -3552,10 +3548,11 @@ class DiscoveredLicense( AbstractLicenseDetection, ): """ - A project's Discovered Licenses are the unique License Detection objects + A project's Discovered Licenses are the unique License Detection objects discovered in the code under analysis. """ + license_expression_field = "license_expression" # If this license was discovered in a extracted license statement @@ -3573,10 +3570,10 @@ class DiscoveredLicense( default=list, blank=True, help_text=_( - 'A list of file regions with resource path, start and end line ' - 'details for each place this license detection was discovered at. ' - 'Also contains whether this license was discovered from a file or ' - 'from package metadata.' + "A list of file regions with resource path, start and end line " + "details for each place this license detection was discovered at. " + "Also contains whether this license was discovered from a file or " + "from package metadata." ), ) @@ -3604,9 +3601,10 @@ def __str__(self): @classmethod def create_from_data(cls, project, detection_data): """ - Create and returns a DiscoveredLicense for a `project` from the `detection_data`. - If one of the values of the required fields is not available, a "ProjectMessage" - is created instead of a new DiscoveredLicense instance. + Create and returns a DiscoveredLicense for a `project` from the + `detection_data`. If one of the values of the required fields is not + available, a "ProjectMessage" is created instead of a new + DiscoveredLicense instance. """ detection_data = detection_data.copy() required_fields = ["license_expression", "identifier", "matches"] @@ -3622,7 +3620,11 @@ def create_from_data(cls, project, detection_data): f"{', '.join(missing_values)}" ) - project.add_warning(description=message, model=cls, details=detection_data) + project.add_warning( + description=message, + model=cls, + details=detection_data, + ) return cleaned_data = { @@ -3633,8 +3635,8 @@ def create_from_data(cls, project, detection_data): discovered_license = cls(project=project, **cleaned_data) # Using save_error=False to not capture potential errors at this level but - # rather in the CodebaseResource.create_and_add_license_data method so resource data - # can be injected in the ProjectMessage record. + # rather in the CodebaseResource.create_and_add_license_data method so + # resource data can be injected in the ProjectMessage record. discovered_license.save(save_error=False, capture_exception=False) return discovered_license @@ -3644,7 +3646,7 @@ def update_with_file_region(self, file_region): `file_regions` list and increase the `detection_count` by 1. """ file_region_data = file_region.to_dict() - if not file_region_data in self.file_regions: + if file_region_data not in self.file_regions: self.file_regions.append(file_region_data) if not self.detection_count: self.detection_count = 1 diff --git a/scanpipe/pipes/__init__.py b/scanpipe/pipes/__init__.py index 3a1bea41e5..96d2bf7b32 100644 --- a/scanpipe/pipes/__init__.py +++ b/scanpipe/pipes/__init__.py @@ -36,8 +36,8 @@ from scanpipe.models import CodebaseRelation from scanpipe.models import CodebaseResource from scanpipe.models import DiscoveredDependency -from scanpipe.models import DiscoveredPackage from scanpipe.models import DiscoveredLicense +from scanpipe.models import DiscoveredPackage from scanpipe.pipes import scancode logger = logging.getLogger("scanpipe.pipes") @@ -247,7 +247,10 @@ def update_or_create_dependency( def update_or_create_license_detection( - project, detection_data, resource_path, from_package=False, + project, + detection_data, + resource_path, + from_package=False, ): """ Get, update or create a DiscoveredLicense object then return it. diff --git a/scanpipe/pipes/input.py b/scanpipe/pipes/input.py index 41e5eb16af..3035ef907e 100644 --- a/scanpipe/pipes/input.py +++ b/scanpipe/pipes/input.py @@ -34,8 +34,8 @@ from scanpipe.models import CodebaseRelation from scanpipe.models import CodebaseResource from scanpipe.models import DiscoveredDependency -from scanpipe.models import DiscoveredPackage from scanpipe.models import DiscoveredLicense +from scanpipe.models import DiscoveredPackage from scanpipe.pipes import scancode from scanpipe.pipes.output import mappings_key_by_fieldname diff --git a/scanpipe/pipes/scancode.py b/scanpipe/pipes/scancode.py index aadc4a62cd..327d3c6d15 100644 --- a/scanpipe/pipes/scancode.py +++ b/scanpipe/pipes/scancode.py @@ -37,9 +37,9 @@ from commoncode import fileutils from commoncode.resource import VirtualCodebase from extractcode import api as extractcode_api +from licensedcode.detection import FileRegion from packagedcode import get_package_handler from packagedcode import models as packagedcode_models -from licensedcode.detection import FileRegion from scancode import Scanner from scancode import api as scancode_api from scancode import cli as scancode_cli @@ -460,12 +460,8 @@ def get_file_region(detection_data, resource_path): object containing information about where this license was detected exactly in a codebase, with `resource_path`, with start and end lines. """ - start_line = min( - [match['start_line'] for match in detection_data["matches"]] - ) - end_line = max( - [match['end_line'] for match in detection_data["matches"]] - ) + start_line = min([match["start_line"] for match in detection_data["matches"]]) + end_line = max([match["end_line"] for match in detection_data["matches"]]) return FileRegion( path=resource_path, start_line=start_line, @@ -694,7 +690,7 @@ def create_discovered_licenses(project, scanned_codebase): """ Save the license detections of a ScanCode `scanned_codebase` scancode.resource.Codebase object to the database as a DiscoveredLicense of - `project`. + `project`. """ if hasattr(scanned_codebase.attributes, "license_detections"): for detection_data in scanned_codebase.attributes.license_detections: diff --git a/scanpipe/views.py b/scanpipe/views.py index ec0afe5fb9..dc2badd41f 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -64,8 +64,8 @@ from scanpipe.api.serializers import DiscoveredDependencySerializer from scanpipe.filters import PAGE_VAR from scanpipe.filters import DependencyFilterSet -from scanpipe.filters import PackageFilterSet from scanpipe.filters import LicenseFilterSet +from scanpipe.filters import PackageFilterSet from scanpipe.filters import ProjectFilterSet from scanpipe.filters import ProjectMessageFilterSet from scanpipe.filters import RelationFilterSet @@ -82,8 +82,8 @@ from scanpipe.models import CodebaseRelation from scanpipe.models import CodebaseResource from scanpipe.models import DiscoveredDependency -from scanpipe.models import DiscoveredPackage from scanpipe.models import DiscoveredLicense +from scanpipe.models import DiscoveredPackage from scanpipe.models import InputSource from scanpipe.models import Project from scanpipe.models import ProjectMessage @@ -2009,7 +2009,7 @@ def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context["dependency_data"] = DiscoveredDependencySerializer(self.object).data return context - + class DiscoveredLicenseDetailsView( ConditionalLoginRequired, From a025a5e307fafbe7390f4ffb8e6e31abb3d72e02 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Mon, 25 Mar 2024 21:19:56 +0530 Subject: [PATCH 06/14] Reposition migrations after rebasing main Signed-off-by: Ayan Sinha Mahapatra --- ...ered_license_models.py => 0056_discovered_license_models.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename scanpipe/migrations/{0054_discovered_license_models.py => 0056_discovered_license_models.py} (98%) diff --git a/scanpipe/migrations/0054_discovered_license_models.py b/scanpipe/migrations/0056_discovered_license_models.py similarity index 98% rename from scanpipe/migrations/0054_discovered_license_models.py rename to scanpipe/migrations/0056_discovered_license_models.py index d4253eb6f8..61511d46bf 100644 --- a/scanpipe/migrations/0054_discovered_license_models.py +++ b/scanpipe/migrations/0056_discovered_license_models.py @@ -8,7 +8,7 @@ class Migration(migrations.Migration): dependencies = [ - ("scanpipe", "0053_restructure_pipelines_data"), + ("scanpipe", "0055_discoveredpackage_datafile_paths"), ] operations = [ From b20ce50416b4285e1ff1987bfcb6061862063788 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 28 Mar 2024 02:13:42 +0530 Subject: [PATCH 07/14] Fix SCIO tests Signed-off-by: Ayan Sinha Mahapatra --- scanpipe/pipes/__init__.py | 14 +++++++++++--- scanpipe/tests/test_models.py | 1 + scanpipe/tests/test_views.py | 2 +- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/scanpipe/pipes/__init__.py b/scanpipe/pipes/__init__.py index 96d2bf7b32..a4bd722f3a 100644 --- a/scanpipe/pipes/__init__.py +++ b/scanpipe/pipes/__init__.py @@ -249,7 +249,7 @@ def update_or_create_dependency( def update_or_create_license_detection( project, detection_data, - resource_path, + resource_path=None, from_package=False, ): """ @@ -277,6 +277,14 @@ def update_or_create_license_detection( detection_data, ) + if not license_detection: + project.add_error( + model="update_or_create_license_detection", + details=detection_data, + resource=resource_path, + ) + return + if resource_path: file_region = scancode.get_file_region( detection_data=detection_data, @@ -290,8 +298,8 @@ def update_or_create_license_detection( def _clean_license_detection_data(detection_data): detection_data = detection_data.copy() - if "sample_matches" in detection_data: - matches = detection_data.pop("sample_matches") + if "reference_matches" in detection_data: + matches = detection_data.pop("reference_matches") detection_data["matches"] = matches return detection_data diff --git a/scanpipe/tests/test_models.py b/scanpipe/tests/test_models.py index 50cb751208..7fc71cb52e 100644 --- a/scanpipe/tests/test_models.py +++ b/scanpipe/tests/test_models.py @@ -168,6 +168,7 @@ def test_scanpipe_project_model_delete_related_objects(self): "scanpipe.CodebaseRelation": 0, "scanpipe.CodebaseResource": 1, "scanpipe.DiscoveredDependency": 0, + "scanpipe.DiscoveredLicense": 0, "scanpipe.DiscoveredPackage": 1, "scanpipe.DiscoveredPackage_codebase_resources": 1, "scanpipe.ProjectMessage": 0, diff --git a/scanpipe/tests/test_views.py b/scanpipe/tests/test_views.py index 554a812b7e..491f816ed9 100644 --- a/scanpipe/tests/test_views.py +++ b/scanpipe/tests/test_views.py @@ -683,7 +683,7 @@ def test_scanpipe_views_project_views(self): with self.assertNumQueries(8): self.client.get(url) - with self.assertNumQueries(13): + with self.assertNumQueries(14): self.client.get(self.project1.get_absolute_url()) @mock.patch("scanpipe.models.Run.execute_task_async") From 3cb1a2cc52cc2d2a77d17ca56271cc6896a4334e Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 28 Mar 2024 16:43:50 +0530 Subject: [PATCH 08/14] Improve detail views with links in UI This commit improves license, package and resource details views with detection tabs and links to each other. Signed-off-by: Ayan Sinha Mahapatra --- scanpipe/models.py | 12 ++- scanpipe/pipes/__init__.py | 16 ++- scanpipe/pipes/scancode.py | 34 +++++- .../panels/resource_license_summary.html | 2 +- .../scanpipe/tabset/tab_detections.html | 34 ------ .../tabset/tab_license_detections.html | 88 +++++++++++++++ .../tabset/tab_package_detections.html | 87 +++++++++++++++ .../tabset/tab_resource_detections.html | 102 ++++++++++++++++++ scanpipe/tests/test_views.py | 12 +-- scanpipe/views.py | 37 ++++--- 10 files changed, 357 insertions(+), 67 deletions(-) delete mode 100644 scanpipe/templates/scanpipe/tabset/tab_detections.html create mode 100644 scanpipe/templates/scanpipe/tabset/tab_license_detections.html create mode 100644 scanpipe/templates/scanpipe/tabset/tab_package_detections.html create mode 100644 scanpipe/templates/scanpipe/tabset/tab_resource_detections.html diff --git a/scanpipe/models.py b/scanpipe/models.py index 52759901d3..c7d0bf513e 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -3640,7 +3640,7 @@ def create_from_data(cls, project, detection_data): discovered_license.save(save_error=False, capture_exception=False) return discovered_license - def update_with_file_region(self, file_region): + def update_with_file_region(self, file_region, count_detection): """ If the `file_region` is a new file region, include it in the `file_regions` list and increase the `detection_count` by 1. @@ -3648,10 +3648,12 @@ def update_with_file_region(self, file_region): file_region_data = file_region.to_dict() if file_region_data not in self.file_regions: self.file_regions.append(file_region_data) - if not self.detection_count: - self.detection_count = 1 - else: - self.detection_count += 1 + if count_detection: + if not self.detection_count: + self.detection_count = 1 + else: + self.detection_count += 1 + self.save(update_fields=["detection_count", "file_regions"]) diff --git a/scanpipe/pipes/__init__.py b/scanpipe/pipes/__init__.py index a4bd722f3a..a48a818b8d 100644 --- a/scanpipe/pipes/__init__.py +++ b/scanpipe/pipes/__init__.py @@ -251,6 +251,7 @@ def update_or_create_license_detection( detection_data, resource_path=None, from_package=False, + count_detection=True, ): """ Get, update or create a DiscoveredLicense object then return it. @@ -290,7 +291,10 @@ def update_or_create_license_detection( detection_data=detection_data, resource_path=resource_path, ) - license_detection.update_with_file_region(file_region) + license_detection.update_with_file_region( + file_region=file_region, + count_detection=count_detection, + ) license_detection.from_package = from_package return license_detection @@ -301,6 +305,16 @@ def _clean_license_detection_data(detection_data): if "reference_matches" in detection_data: matches = detection_data.pop("reference_matches") detection_data["matches"] = matches + + updated_matches = [] + for match_data in detection_data["matches"]: + from_file_path = match_data["from_file"] + if from_file_path: + match_data["from_file"] = from_file_path.removeprefix("codebase/") + + updated_matches.append(match_data) + + detection_data["matches"] = updated_matches return detection_data diff --git a/scanpipe/pipes/scancode.py b/scanpipe/pipes/scancode.py index 327d3c6d15..fc818a4038 100644 --- a/scanpipe/pipes/scancode.py +++ b/scanpipe/pipes/scancode.py @@ -647,10 +647,28 @@ def create_codebase_resources(project, scanned_codebase): license_detections = getattr(scanned_resource, "license_detections", []) for detection_data in license_detections: detection_identifier = detection_data.get("identifier") - license_detection = project.discoveredlicenses.get_or_none( - identifier=detection_identifier + pipes.update_or_create_license_detection( + project=project, + detection_data=detection_data, + resource_path=resource_path, + count_detection=False, ) - logger.debug(f"Add {codebase_resource} to {license_detection}") + logger.debug(f"Add {codebase_resource} to {detection_identifier}") + + packages = getattr(scanned_resource, "package_data", []) + for package_data in packages: + license_detections = package_data.get("license_detections", []) + license_detections.extend(package_data.get("other_license_detections", [])) + for detection_data in license_detections: + detection_identifier = detection_data.get("identifier") + pipes.update_or_create_license_detection( + project=project, + detection_data=detection_data, + resource_path=resource_path, + count_detection=False, + from_package=True, + ) + logger.debug(f"Add {codebase_resource} to {detection_identifier}") def create_discovered_packages(project, scanned_codebase): @@ -661,6 +679,16 @@ def create_discovered_packages(project, scanned_codebase): if hasattr(scanned_codebase.attributes, "packages"): for package_data in scanned_codebase.attributes.packages: pipes.update_or_create_package(project, package_data) + license_detections = package_data.get("license_detections", []) + license_detections.extend(package_data.get("other_license_detections", [])) + + for license_detection in license_detections: + pipes.update_or_create_license_detection( + project=project, + detection_data=license_detection, + from_package=True, + count_detection=False, + ) def create_discovered_dependencies( diff --git a/scanpipe/templates/scanpipe/panels/resource_license_summary.html b/scanpipe/templates/scanpipe/panels/resource_license_summary.html index 7a0f723888..c2e1587539 100644 --- a/scanpipe/templates/scanpipe/panels/resource_license_summary.html +++ b/scanpipe/templates/scanpipe/panels/resource_license_summary.html @@ -3,7 +3,7 @@
-
-
-

License Detections

-

- {% if project.license_detections_count %} - - {{ project.license_detections_count|intcomma }} - - {% else %} - 0 - {% endif %} -

-
-

Resources

diff --git a/scanpipe/templates/scanpipe/license_detection_list.html b/scanpipe/templates/scanpipe/license_detection_list.html index 426c5b1452..caedea9aac 100644 --- a/scanpipe/templates/scanpipe/license_detection_list.html +++ b/scanpipe/templates/scanpipe/license_detection_list.html @@ -24,23 +24,28 @@ {# CAUTION: Avoid relying on get_absolute_url to prevent unnecessary query triggers #} {{ license_detection.identifier }} + {% if license_detection.has_compliance_alert %} + + + + {% endif %} - {{ license_detection.license_expression }} + {{ license_detection.license_expression }} - {{ license_detection.license_expression_spdx }} + {{ license_detection.license_expression_spdx }} {{ license_detection.detection_count }} {% if display_compliance_alert %} - - {{ package.compliance_alert }} + + {{ license_detection.compliance_alert }} - {% endif %} + {% endif %} {% empty %} diff --git a/scanpipe/templates/scanpipe/package_list.html b/scanpipe/templates/scanpipe/package_list.html index 8fc0654a11..9c277833de 100644 --- a/scanpipe/templates/scanpipe/package_list.html +++ b/scanpipe/templates/scanpipe/package_list.html @@ -30,6 +30,11 @@ {% endif %} + {% if package.has_compliance_alert %} + + + + {% endif %} diff --git a/scanpipe/templates/scanpipe/panels/license_detections_summary.html b/scanpipe/templates/scanpipe/panels/license_detections_summary.html new file mode 100644 index 0000000000..6e045491fc --- /dev/null +++ b/scanpipe/templates/scanpipe/panels/license_detections_summary.html @@ -0,0 +1,19 @@ +{% load humanize %} +{% if license_detection_summary %} +
+ +
+{% endif %} \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/panels/resource_license_summary.html b/scanpipe/templates/scanpipe/panels/resource_license_summary.html deleted file mode 100644 index c2e1587539..0000000000 --- a/scanpipe/templates/scanpipe/panels/resource_license_summary.html +++ /dev/null @@ -1,16 +0,0 @@ -{% load humanize %} -{% if resource_license_summary %} -
- -
-{% endif %} \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/project_detail.html b/scanpipe/templates/scanpipe/project_detail.html index 3441fea8c1..b398c2a83a 100644 --- a/scanpipe/templates/scanpipe/project_detail.html +++ b/scanpipe/templates/scanpipe/project_detail.html @@ -104,7 +104,7 @@
-
+
{% if project.extra_data %} diff --git a/scanpipe/templates/scanpipe/tabset/tab_packages.html b/scanpipe/templates/scanpipe/tabset/tab_packages.html index 1370d741da..3231d4d865 100644 --- a/scanpipe/templates/scanpipe/tabset/tab_packages.html +++ b/scanpipe/templates/scanpipe/tabset/tab_packages.html @@ -18,6 +18,11 @@ {% endif %} + {% if package.has_compliance_alert %} + + + + {% endif %} {{ package.declared_license_expression }} diff --git a/scanpipe/tests/test_views.py b/scanpipe/tests/test_views.py index 7130bd3d5d..bb31743177 100644 --- a/scanpipe/tests/test_views.py +++ b/scanpipe/tests/test_views.py @@ -683,7 +683,7 @@ def test_scanpipe_views_project_views(self): with self.assertNumQueries(8): self.client.get(url) - with self.assertNumQueries(14): + with self.assertNumQueries(13): self.client.get(self.project1.get_absolute_url()) @mock.patch("scanpipe.models.Run.execute_task_async") @@ -915,7 +915,7 @@ def test_scanpipe_views_codebase_resource_views(self): with self.assertNumQueries(7): self.client.get(url) - with self.assertNumQueries(7): + with self.assertNumQueries(8): self.client.get(resource1.get_absolute_url()) def test_scanpipe_views_discovered_package_views(self): diff --git a/scanpipe/urls.py b/scanpipe/urls.py index 5469c7e1a0..417064c4cd 100644 --- a/scanpipe/urls.py +++ b/scanpipe/urls.py @@ -192,9 +192,9 @@ name="project_resource_status_summary", ), path( - "project//resource_license_summary/", - views.ProjectResourceLicenseSummaryView.as_view(), - name="project_resource_license_summary", + "project//license_detection_summary/", + views.ProjectLicenseDetectionSummaryView.as_view(), + name="project_license_detection_summary", ), path( "project//", diff --git a/scanpipe/views.py b/scanpipe/views.py index 11242acd72..6238e18909 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -944,14 +944,14 @@ def get_context_data(self, **kwargs): return context -class ProjectResourceLicenseSummaryView(ConditionalLoginRequired, generic.DetailView): +class ProjectLicenseDetectionSummaryView(ConditionalLoginRequired, generic.DetailView): model = Project - template_name = "scanpipe/panels/resource_license_summary.html" + template_name = "scanpipe/panels/license_detections_summary.html" @staticmethod - def get_resource_license_summary(project, limit=10): + def get_license_detection_summary(project, limit=10): license_counter = count_group_by( - project.codebaseresources.files(), "detected_license_expression" + project.discoveredlicenses, "license_expression" ) if list(license_counter.keys()) == [""]: @@ -962,24 +962,29 @@ def get_resource_license_summary(project, limit=10): sorted(license_counter.items(), key=operator.itemgetter(1), reverse=True) ) - # Remove the "no licenses" entry from the top list - no_licenses = sorted_by_count.pop("", None) - # Keep the top entries top_licenses = dict(list(sorted_by_count.items())[:limit]) - # Add the "no licenses" entry at the end - if no_licenses: - top_licenses[""] = no_licenses + # Also get count for detections with + expressions_with_compliance_alert = [] + for license_expression in top_licenses.keys(): + has_compliance_alert = ( + project.discoveredlicenses.filter(license_expression=license_expression) + .has_compliance_alert() + .exists() + ) + if has_compliance_alert: + expressions_with_compliance_alert.append(license_expression) - return top_licenses + return top_licenses, expressions_with_compliance_alert def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - summary = self.get_resource_license_summary(project=self.object) - context["resource_license_summary"] = summary - context["project_resources_url"] = reverse( - "project_resources", args=[self.object.slug] + summary, expressions = self.get_license_detection_summary(project=self.object) + context["license_detection_summary"] = summary + context["expressions_with_compliance_alert"] = expressions + context["project_licenses_url"] = reverse( + "project_licenses", args=[self.object.slug] ) return context @@ -1544,7 +1549,10 @@ class DiscoveredLicenseListView( "field_name": "license_expression", "filter_fieldname": "license_expression", }, - "license_expression_spdx", + { + "field_name": "license_expression_spdx", + "filter_fieldname": "license_expression_spdx", + }, "detection_count", { "field_name": "compliance_alert", @@ -1566,6 +1574,11 @@ def get_queryset(self): .order_by_count_and_expression() ) + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context["display_compliance_alert"] = scanpipe_app.policies_enabled + return context + class ProjectMessageListView( ConditionalLoginRequired, From 973d7aa7698eaaeefa20731651b85e4d80443514 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 18 Jul 2024 03:00:03 +0530 Subject: [PATCH 10/14] Improve license detection and summary related views Signed-off-by: Ayan Sinha Mahapatra --- scanpipe/filters.py | 6 +- scanpipe/models.py | 33 +++- .../includes/project_summary_level.html | 8 +- .../scanpipe/license_detection_list.html | 2 +- .../panels/license_detections_summary.html | 18 +- .../scanpipe/panels/scan_summary_panel.html | 162 ++++++++++++++++-- .../tabset/tab_license_detections.html | 4 +- scanpipe/tests/test_views.py | 19 +- scanpipe/views.py | 49 ++++-- 9 files changed, 245 insertions(+), 56 deletions(-) diff --git a/scanpipe/filters.py b/scanpipe/filters.py index b60888af54..307cfbfdcd 100644 --- a/scanpipe/filters.py +++ b/scanpipe/filters.py @@ -492,6 +492,7 @@ class ResourceFilterSet(FilterSetUtilsMixin, django_filters.FilterSet): dropdown_widget_fields = [ "status", "type", + "programming_language", "tag", "compliance_alert", "in_package", @@ -521,6 +522,7 @@ class ResourceFilterSet(FilterSetUtilsMixin, django_filters.FilterSet): "related_from__from_resource__path", ], ) + programming_language = django_filters.AllValuesFilter() compliance_alert = django_filters.ChoiceFilter( choices=[(EMPTY_VAR, "None")] + CodebaseResource.Compliance.choices, ) @@ -577,8 +579,8 @@ class Meta: def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - license_expression_filer = self.filters["detected_license_expression"] - license_expression_filer.extra["widget"] = HasValueDropdownWidget() + license_expression_filter = self.filters["detected_license_expression"] + license_expression_filter.extra["widget"] = HasValueDropdownWidget() @classmethod def filter_for_lookup(cls, field, lookup_type): diff --git a/scanpipe/models.py b/scanpipe/models.py index f47fed0ece..d2aed87ec7 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -1365,14 +1365,28 @@ def license_detections_count(self): @cached_property def package_compliance_alert_count(self): - """Return the number of packages related to this project which have.""" + """ + Return the number of packages related to this project which have + a license compliance error alert. + """ return self.discoveredpackages.has_compliance_alert().count() @cached_property def license_compliance_alert_count(self): - """Return the number of packages related to this project which have.""" + """ + Return the number of license detections related to this project + which have a license compliance error alert. + """ return self.discoveredlicenses.has_compliance_alert().count() + @cached_property + def resource_compliance_alert_count(self): + """ + Return the number of codebase resources related to this project which have + a license compliance error alert. + """ + return self.codebaseresources.has_compliance_alert().count() + @cached_property def message_count(self): """Return the number of messages related to this project.""" @@ -2042,7 +2056,15 @@ def convert_glob_to_django_regex(glob_pattern): return escaped_pattern -class CodebaseResourceQuerySet(ProjectRelatedQuerySet): +class ComplianceAlertQuerySetMixin: + def has_compliance_alert(self): + return self.filter(Q(compliance_alert__exact=CodebaseResource.Compliance.ERROR)) + + +class CodebaseResourceQuerySet( + ComplianceAlertQuerySetMixin, + ProjectRelatedQuerySet, +): def prefetch_for_serializer(self): """ Optimized prefetching for a QuerySet to be consumed by the @@ -2945,11 +2967,6 @@ def vulnerable(self): return self.filter(~Q(affected_by_vulnerabilities__in=EMPTY_VALUES)) -class ComplianceAlertQuerySetMixin: - def has_compliance_alert(self): - return self.filter(Q(compliance_alert__exact=CodebaseResource.Compliance.ERROR)) - - class DiscoveredPackageQuerySet( VulnerabilityQuerySetMixin, ComplianceAlertQuerySetMixin, diff --git a/scanpipe/templates/scanpipe/includes/project_summary_level.html b/scanpipe/templates/scanpipe/includes/project_summary_level.html index 6ab505e55f..e5c36665bd 100644 --- a/scanpipe/templates/scanpipe/includes/project_summary_level.html +++ b/scanpipe/templates/scanpipe/includes/project_summary_level.html @@ -49,11 +49,17 @@

Resources

-

+

{% if project.resource_count %} {{ project.resource_count|intcomma }} + {% if project.resource_compliance_alert_count %} + + {{ project.resource_compliance_alert_count|intcomma }} + + + {% endif %} {% else %} 0 {% endif %} diff --git a/scanpipe/templates/scanpipe/license_detection_list.html b/scanpipe/templates/scanpipe/license_detection_list.html index caedea9aac..ea04ab67bc 100644 --- a/scanpipe/templates/scanpipe/license_detection_list.html +++ b/scanpipe/templates/scanpipe/license_detection_list.html @@ -26,7 +26,7 @@ {{ license_detection.identifier }} {% if license_detection.has_compliance_alert %} - + {% endif %} diff --git a/scanpipe/templates/scanpipe/panels/license_detections_summary.html b/scanpipe/templates/scanpipe/panels/license_detections_summary.html index 6e045491fc..6b71f5ed49 100644 --- a/scanpipe/templates/scanpipe/panels/license_detections_summary.html +++ b/scanpipe/templates/scanpipe/panels/license_detections_summary.html @@ -3,17 +3,29 @@

{% endif %} \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/panels/scan_summary_panel.html b/scanpipe/templates/scanpipe/panels/scan_summary_panel.html index 52c11c1a22..1def8b4be5 100644 --- a/scanpipe/templates/scanpipe/panels/scan_summary_panel.html +++ b/scanpipe/templates/scanpipe/panels/scan_summary_panel.html @@ -6,15 +6,80 @@
- {% for field_label, values in scan_summary.items %} - - - + + + + + + + + + + + + + + - - {% endfor %} + + {% endif %} + {% endfor %} + + + + + + + + + + + + + + +
- {{ field_label }} - -
    - {% for entry in values %} - {% if entry.value %} +
+ Declared license + +
    + {% for entry in scan_summary.declared_license_expression %} + {% if entry.value %} +
  • + {{ entry.value }} + {% if entry.count %} + + {{ entry.count|intcomma }} + + {% endif %} +
    • + {% endif %} + {% endfor %} +
+
+ Declared holder + +
    + {% for entry in scan_summary.declared_holder %} + {% if entry.value %} +
  • + {{ entry.value }} + {% if entry.count %} + + {{ entry.count|intcomma }} + + {% endif %} +
    • + {% endif %} + {% endfor %} +
+
+ Primary language + + +
+ Other licenses + + -
+ Other holders + +
    + {% for entry in scan_summary.other_holders %} + {% if entry.value %} +
  • + {{ entry.value }} + {% if entry.count %} + + {{ entry.count|intcomma }} + + {% endif %} +
    • + {% endif %} + {% endfor %} +
+
+ Other languages + + +
+ Key Files + + +
\ No newline at end of file diff --git a/scanpipe/templates/scanpipe/tabset/tab_license_detections.html b/scanpipe/templates/scanpipe/tabset/tab_license_detections.html index c1fd375d7c..e5b6fcd2e6 100644 --- a/scanpipe/templates/scanpipe/tabset/tab_license_detections.html +++ b/scanpipe/templates/scanpipe/tabset/tab_license_detections.html @@ -19,7 +19,7 @@ {{ match.license_expression }} - {{ match.from_file }} + {{ match.from_file }} {{ match.rule_url }} @@ -73,7 +73,7 @@ {% for file_region in tab_data.fields.file_regions.value %} - {{ file_region.path }} + {{ file_region.path }} {{ file_region.start_line }} diff --git a/scanpipe/tests/test_views.py b/scanpipe/tests/test_views.py index 5c762b325e..1282105018 100644 --- a/scanpipe/tests/test_views.py +++ b/scanpipe/tests/test_views.py @@ -475,16 +475,17 @@ def test_scanpipe_views_project_details_get_scan_summary_data(self): scan_summary = self.data / "scancode" / "is-npm-1.0.0_scan_package_summary.json" scan_summary_json = json.loads(scan_summary.read_text()) - scan_summary_data = get_scan_summary_data(scan_summary_json) + scan_summary_data = get_scan_summary_data(self.project1, scan_summary_json) - self.assertEqual(6, len(scan_summary_data)) + self.assertEqual(7, len(scan_summary_data)) expected = [ - "Declared license", - "Declared holder", - "Primary language", - "Other licenses", - "Other holders", - "Other languages", + "declared_license_expression", + "declared_holder", + "primary_language", + "other_license_expressions", + "other_holders", + "other_languages", + "key_file_licenses", ] self.assertEqual(expected, list(scan_summary_data.keys())) @@ -956,7 +957,7 @@ def test_scanpipe_views_codebase_resource_views(self): package1.add_resources([resource1, resource2]) url = reverse("project_resources", args=[self.project1.slug]) - with self.assertNumQueries(8): + with self.assertNumQueries(9): self.client.get(url) with self.assertNumQueries(8): diff --git a/scanpipe/views.py b/scanpipe/views.py index 00c5976904..8df4d1bc03 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -173,12 +173,12 @@ SCAN_SUMMARY_FIELDS = [ - ("Declared license", "declared_license_expression"), - ("Declared holder", "declared_holder"), - ("Primary language", "primary_language"), - ("Other licenses", "other_license_expressions"), - ("Other holders", "other_holders"), - ("Other languages", "other_languages"), + "declared_license_expression", + "declared_holder", + "primary_language", + "other_license_expressions", + "other_holders", + "other_languages", ] @@ -317,7 +317,9 @@ def get_field_value(self, field_name, render_func=None): """ Return the formatted value of the specified `field_name` from the object. - By default, JSON types (list and dict) are rendered as YAML. + By default, JSON types (list and dict) are rendered as YAML, + except some fields which are used for a more complex tabular + representation with links to other views. If a `render_func` is provided, it will take precedence and be used for rendering the value. """ @@ -652,11 +654,12 @@ def get_license_clarity_data(scan_summary_json): ] @staticmethod - def get_scan_summary_data(scan_summary_json): + def get_scan_summary_data(project, scan_summary_json): summary_data = {} - for field_label, field_name in SCAN_SUMMARY_FIELDS: - field_data = scan_summary_json.get(field_name) + for field_name, field_data in scan_summary_json.items(): + if field_name not in SCAN_SUMMARY_FIELDS: + continue if type(field_data) is list: # Do not include `None` entries @@ -665,7 +668,13 @@ def get_scan_summary_data(scan_summary_json): # Converts single value type into common data-structure values = [{"value": field_data}] - summary_data[field_label] = values + summary_data[field_name] = values + + key_files = project.codebaseresources.filter(is_key_file=True) + summary_data["key_file_licenses"] = { + key_file.path: key_file.detected_license_expression + for key_file in key_files + } return summary_data @@ -723,7 +732,7 @@ def get_context_data(self, **kwargs): with suppress(json.decoder.JSONDecodeError): scan_summary_json = json.loads(scan_summary_file.read_text()) license_clarity = self.get_license_clarity_data(scan_summary_json) - scan_summary = self.get_scan_summary_data(scan_summary_json) + scan_summary = self.get_scan_summary_data(project, scan_summary_json) codebase_root = sorted( project.codebase_path.glob("*"), @@ -982,7 +991,7 @@ def get_license_detection_summary(project, limit=10): ) if list(license_counter.keys()) == [""]: - return + return None, None, None # Order the license list by the number of detections, higher first sorted_by_count = dict( @@ -1003,13 +1012,23 @@ def get_license_detection_summary(project, limit=10): if has_compliance_alert: expressions_with_compliance_alert.append(license_expression) - return top_licenses, expressions_with_compliance_alert + total_counts = { + "with_compliance_error": ( + project.discoveredlicenses.has_compliance_alert().count() + ), + "all": project.discoveredlicenses.count(), + } + + return top_licenses, expressions_with_compliance_alert, total_counts def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - summary, expressions = self.get_license_detection_summary(project=self.object) + summary, expressions, counts = self.get_license_detection_summary( + project=self.object + ) context["license_detection_summary"] = summary context["expressions_with_compliance_alert"] = expressions + context["total_counts"] = counts context["project_licenses_url"] = reverse( "project_licenses", args=[self.object.slug] ) From 9f220accaa275c8a3fcb52ad82533e1ec5e4b133 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Sat, 7 Sep 2024 00:43:33 +0530 Subject: [PATCH 11/14] Add license detection improvements Signed-off-by: Ayan Sinha Mahapatra --- .../scanpipe/license_detection_list.html | 2 +- .../panels/license_detections_summary.html | 4 +-- .../tabset/tab_license_detections.html | 31 +++++++++++++------ .../tabset/tab_package_detections.html | 14 ++++----- .../tabset/tab_resource_detections.html | 18 +++++------ 5 files changed, 40 insertions(+), 29 deletions(-) diff --git a/scanpipe/templates/scanpipe/license_detection_list.html b/scanpipe/templates/scanpipe/license_detection_list.html index ea04ab67bc..6b5ced7892 100644 --- a/scanpipe/templates/scanpipe/license_detection_list.html +++ b/scanpipe/templates/scanpipe/license_detection_list.html @@ -50,7 +50,7 @@ {% empty %} - No Licenses detected. Clear search and filters + No licenses detected. Clear search and filters {% endfor %} diff --git a/scanpipe/templates/scanpipe/panels/license_detections_summary.html b/scanpipe/templates/scanpipe/panels/license_detections_summary.html index 6b71f5ed49..91bba80863 100644 --- a/scanpipe/templates/scanpipe/panels/license_detections_summary.html +++ b/scanpipe/templates/scanpipe/panels/license_detections_summary.html @@ -3,7 +3,7 @@