From 4d0b4d5b2e4e611b6ce27897dfaa8d370d2e7532 Mon Sep 17 00:00:00 2001 From: Aayush Kumar Date: Fri, 27 Jun 2025 18:23:59 +0530 Subject: [PATCH 01/21] add left-pane file tree view and related templates Signed-off-by: Aayush Kumar --- .../scanpipe/panels/file_tree_panel.html | 42 ++++++++++++ .../templates/scanpipe/resource_tree.html | 67 +++++++++++++++++++ scanpipe/tests/test_views.py | 32 +++++++++ scanpipe/urls.py | 5 ++ scanpipe/views.py | 34 ++++++++++ 5 files changed, 180 insertions(+) create mode 100644 scanpipe/templates/scanpipe/panels/file_tree_panel.html create mode 100644 scanpipe/templates/scanpipe/resource_tree.html diff --git a/scanpipe/templates/scanpipe/panels/file_tree_panel.html b/scanpipe/templates/scanpipe/panels/file_tree_panel.html new file mode 100644 index 0000000000..00ebbe79af --- /dev/null +++ b/scanpipe/templates/scanpipe/panels/file_tree_panel.html @@ -0,0 +1,42 @@ + diff --git a/scanpipe/templates/scanpipe/resource_tree.html b/scanpipe/templates/scanpipe/resource_tree.html new file mode 100644 index 0000000000..c43698404f --- /dev/null +++ b/scanpipe/templates/scanpipe/resource_tree.html @@ -0,0 +1,67 @@ +{% extends "scanpipe/base.html" %} +{% load static %} +{% block title %}ScanCode.io: {{ project.name }} - Resource{% endblock %} + +{% block extrahead %} + +{% endblock %} + +{% block content %} +
+
+
+ {% include "scanpipe/panels/file_tree_panel.html" with children=children path=path %} +
+
+ +
+
+
+
+
+{% endblock %} + +{% block scripts %} + +{% endblock %} \ No newline at end of file diff --git a/scanpipe/tests/test_views.py b/scanpipe/tests/test_views.py index e7f33091fa..1907d5c5a1 100644 --- a/scanpipe/tests/test_views.py +++ b/scanpipe/tests/test_views.py @@ -1630,3 +1630,35 @@ def test_project_codebase_resources_export_json(self): for field in expected_fields: self.assertIn(field, json_data[0]) + + def test_file_tree_base_url_lists_top_level_nodes(self): + make_resource_file(self.project1, path="child1.txt") + make_resource_file(self.project1, path="dir1") + + url = reverse("file_tree", kwargs={"slug": self.project1.slug}) + response = self.client.get(url) + children = response.context[-1]["children"] + + child1 = children[0] + dir1 = children[1] + + self.assertEqual(child1.path, "child1.txt") + self.assertEqual(dir1.path, "dir1") + + def test_file_tree_nested_url_lists_only_children_of_given_path(self): + make_resource_file(self.project1, path="parent/child1.txt") + make_resource_file(self.project1, path="parent/dir1") + make_resource_file(self.project1, path="parent/dir1/child2.txt") + + url = reverse("file_tree", kwargs={"slug": self.project1.slug}) + response = self.client.get(url + "?path=parent&tree=true") + children = response.context["children"] + + child1 = children[0] + dir1 = children[1] + + self.assertEqual(child1.path, "parent/child1.txt") + self.assertEqual(dir1.path, "parent/dir1") + + self.assertFalse(child1.has_children) + self.assertTrue(dir1.has_children) diff --git a/scanpipe/urls.py b/scanpipe/urls.py index 5efe6fdd76..15d91001d2 100644 --- a/scanpipe/urls.py +++ b/scanpipe/urls.py @@ -251,5 +251,10 @@ views.LicenseListView.as_view(), name="license_list", ), + path( + "project//codebase_tree/", + views.CodebaseResourceTreeView.as_view(), + name="file_tree", + ), path("monitor/", include("django_rq.urls")), ] diff --git a/scanpipe/views.py b/scanpipe/views.py index d52eafcaa2..b1e3b34720 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -37,6 +37,8 @@ from django.core.exceptions import ValidationError from django.core.files.storage.filesystem import FileSystemStorage from django.core.serializers.json import DjangoJSONEncoder +from django.db.models import Exists +from django.db.models import OuterRef from django.db.models import Prefetch from django.db.models.manager import Manager from django.http import FileResponse @@ -2756,3 +2758,35 @@ def get_node(self, package): "children": children, } return node + + +class CodebaseResourceTreeView(ConditionalLoginRequired, generic.DetailView): + template_name = "scanpipe/resource_tree.html" + + def get(self, request, *args, **kwargs): + slug = self.kwargs.get("slug") + project = get_object_or_404(Project, slug=slug) + path = request.GET.get("path", None) + + base_qs = ( + CodebaseResource.objects.filter(project=project, parent_path=path) + .only("path", "name", "type") + .order_by("path") + ) + + subdirs = CodebaseResource.objects.filter( + project=project, + parent_path=OuterRef("path"), + ) + + children = base_qs.annotate(has_children=Exists(subdirs)) + + context = { + "project": project, + "path": path, + "children": children, + } + + if request.GET.get("tree") == "true": + return render(request, "scanpipe/panels/file_tree_panel.html", context) + return render(request, self.template_name, context) From 8e5dd0c5f115ea6f233f28cd4848190e459bdb6a Mon Sep 17 00:00:00 2001 From: Aayush Kumar Date: Fri, 27 Jun 2025 18:47:27 +0530 Subject: [PATCH 02/21] temporarily include parent_path field from previous pr for tests Signed-off-by: Aayush Kumar --- ...3_codebaseresource_parent_path_and_more.py | 24 +++++++++++++++++++ scanpipe/models.py | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 scanpipe/migrations/0073_codebaseresource_parent_path_and_more.py diff --git a/scanpipe/migrations/0073_codebaseresource_parent_path_and_more.py b/scanpipe/migrations/0073_codebaseresource_parent_path_and_more.py new file mode 100644 index 0000000000..86848b879f --- /dev/null +++ b/scanpipe/migrations/0073_codebaseresource_parent_path_and_more.py @@ -0,0 +1,24 @@ +# Generated by Django 5.1.9 on 2025-06-19 21:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('scanpipe', '0072_discovereddependency_uuid_unique'), + ] + + operations = [ + migrations.AddField( + model_name='codebaseresource', + name='parent_path', + field=models.CharField(blank=True, help_text="The path of the resource's parent directory. Set to None for top-level (root) resources. Used to efficiently retrieve a directory's contents.", max_length=2000, null=True), + ), + migrations.AddIndex( + model_name='codebaseresource', + index=models.Index(fields=['project', 'parent_path'], name='scanpipe_co_project_008448_idx'), + ), + ] + + diff --git a/scanpipe/models.py b/scanpipe/models.py index 0ecd8e6392..465ba97436 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -231,7 +231,7 @@ def delete(self, *args, **kwargs): Note that projects with queued or running pipeline runs cannot be deleted. See the `_raise_if_run_in_progress` method. The following if statements should not be triggered unless the `.delete()` - method is directly call from an instance of this class. + method is directly call from a instance of this class. """ with suppress(redis.exceptions.ConnectionError, AttributeError): if self.status == self.Status.RUNNING: From 606c2b6750a82fa12724a32cada8807b19ac3749 Mon Sep 17 00:00:00 2001 From: Aayush Kumar Date: Thu, 3 Jul 2025 22:03:16 +0530 Subject: [PATCH 03/21] Some formatting changes Signed-off-by: Aayush Kumar --- .../scanpipe/panels/file_tree_panel.html | 37 +++---- .../templates/scanpipe/resource_tree.html | 97 ++++++++++--------- 2 files changed, 62 insertions(+), 72 deletions(-) diff --git a/scanpipe/templates/scanpipe/panels/file_tree_panel.html b/scanpipe/templates/scanpipe/panels/file_tree_panel.html index 00ebbe79af..95da0dd6c2 100644 --- a/scanpipe/templates/scanpipe/panels/file_tree_panel.html +++ b/scanpipe/templates/scanpipe/panels/file_tree_panel.html @@ -2,38 +2,25 @@ {% for node in children %}
  • {% if node.is_dir %} -
    - - - - +
    + + + + + {{ node.name }}
    - {% if node.has_children %} {% endif %} - {% else %} -
    - +
    + + + {{ node.name }}
    {% endif %} diff --git a/scanpipe/templates/scanpipe/resource_tree.html b/scanpipe/templates/scanpipe/resource_tree.html index c43698404f..03a1f4a600 100644 --- a/scanpipe/templates/scanpipe/resource_tree.html +++ b/scanpipe/templates/scanpipe/resource_tree.html @@ -1,33 +1,39 @@ {% extends "scanpipe/base.html" %} -{% load static %} -{% block title %}ScanCode.io: {{ project.name }} - Resource{% endblock %} +{% load static humanize %} +{% block title %}ScanCode.io: {{ project.name }} - Resource Tree{% endblock %} {% block extrahead %} - + {% endblock %} {% block content %} -
    -
    +
    + {% include 'scanpipe/includes/navbar_header.html' %} +
    +
    + {% include 'scanpipe/includes/breadcrumb.html' with linked_project=True current="Resource Tree" %} +
    +
    +
    + +
    +
    {% include "scanpipe/panels/file_tree_panel.html" with children=children path=path %}
    - -
    +
    @@ -35,33 +41,30 @@ {% endblock %} {% block scripts %} - + chevron.classList.toggle("rotated"); + e.stopPropagation(); + return; + } + }); + {% endblock %} \ No newline at end of file From 8a5b576acc9e539c0e4f1bde0bb53c9b552121d5 Mon Sep 17 00:00:00 2001 From: Aayush Kumar Date: Fri, 4 Jul 2025 10:14:26 +0530 Subject: [PATCH 04/21] bump migration up to resolve failing tests Signed-off-by: Aayush Kumar --- ...4_codebaseresource_parent_path_and_more.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 scanpipe/migrations/0074_codebaseresource_parent_path_and_more.py diff --git a/scanpipe/migrations/0074_codebaseresource_parent_path_and_more.py b/scanpipe/migrations/0074_codebaseresource_parent_path_and_more.py new file mode 100644 index 0000000000..0f424f678b --- /dev/null +++ b/scanpipe/migrations/0074_codebaseresource_parent_path_and_more.py @@ -0,0 +1,24 @@ +# Generated by Django 5.1.9 on 2025-06-19 21:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('scanpipe', '0073_add_sha1_git_checksum'), + ] + + operations = [ + migrations.AddField( + model_name='codebaseresource', + name='parent_path', + field=models.CharField(blank=True, help_text="The path of the resource's parent directory. Set to None for top-level (root) resources. Used to efficiently retrieve a directory's contents.", max_length=2000, null=True), + ), + migrations.AddIndex( + model_name='codebaseresource', + index=models.Index(fields=['project', 'parent_path'], name='scanpipe_co_project_008448_idx'), + ), + ] + + From 69ad5ac8c7ad6789961dc0b05eaf8ddb744f1e89 Mon Sep 17 00:00:00 2001 From: Aayush Kumar Date: Fri, 4 Jul 2025 10:19:22 +0530 Subject: [PATCH 05/21] remove conflicting migration Signed-off-by: Aayush Kumar --- ...3_codebaseresource_parent_path_and_more.py | 24 ------------------- 1 file changed, 24 deletions(-) delete mode 100644 scanpipe/migrations/0073_codebaseresource_parent_path_and_more.py diff --git a/scanpipe/migrations/0073_codebaseresource_parent_path_and_more.py b/scanpipe/migrations/0073_codebaseresource_parent_path_and_more.py deleted file mode 100644 index 86848b879f..0000000000 --- a/scanpipe/migrations/0073_codebaseresource_parent_path_and_more.py +++ /dev/null @@ -1,24 +0,0 @@ -# Generated by Django 5.1.9 on 2025-06-19 21:19 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('scanpipe', '0072_discovereddependency_uuid_unique'), - ] - - operations = [ - migrations.AddField( - model_name='codebaseresource', - name='parent_path', - field=models.CharField(blank=True, help_text="The path of the resource's parent directory. Set to None for top-level (root) resources. Used to efficiently retrieve a directory's contents.", max_length=2000, null=True), - ), - migrations.AddIndex( - model_name='codebaseresource', - index=models.Index(fields=['project', 'parent_path'], name='scanpipe_co_project_008448_idx'), - ), - ] - - From 2c1dd0f682f8dcfc229f85eda6be5f3cf9e90613 Mon Sep 17 00:00:00 2001 From: Aayush Kumar Date: Tue, 8 Jul 2025 00:41:26 +0530 Subject: [PATCH 06/21] implement suggested changes Signed-off-by: Aayush Kumar --- scanpipe/models.py | 13 +++++++++++++ ...ile_tree_panel.html => codebase_tree_panel.html} | 8 ++++---- scanpipe/templates/scanpipe/resource_tree.html | 11 ++++------- scanpipe/urls.py | 10 +++++----- scanpipe/views.py | 11 ++--------- 5 files changed, 28 insertions(+), 25 deletions(-) rename scanpipe/templates/scanpipe/panels/{file_tree_panel.html => codebase_tree_panel.html} (72%) diff --git a/scanpipe/models.py b/scanpipe/models.py index 465ba97436..91cad133eb 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -46,6 +46,7 @@ from django.db import transaction from django.db.models import Case from django.db.models import Count +from django.db.models import Exists from django.db.models import IntegerField from django.db.models import OuterRef from django.db.models import Prefetch @@ -2425,6 +2426,18 @@ def macho_binaries(self): def executable_binaries(self): return self.union(self.win_exes(), self.macho_binaries(), self.elfs()) + def with_children(self, project): + """ + Annotate the QuerySet with has_children field based on whether + each resource has any children (subdirectories/files). + """ + subdirs = CodebaseResource.objects.filter( + project=project, + parent_path=OuterRef("path"), + ) + + return self.annotate(has_children=Exists(subdirs)) + class ScanFieldsModelMixin(models.Model): """Fields returned by the ScanCode-toolkit scans.""" diff --git a/scanpipe/templates/scanpipe/panels/file_tree_panel.html b/scanpipe/templates/scanpipe/panels/codebase_tree_panel.html similarity index 72% rename from scanpipe/templates/scanpipe/panels/file_tree_panel.html rename to scanpipe/templates/scanpipe/panels/codebase_tree_panel.html index 95da0dd6c2..0ccc12b538 100644 --- a/scanpipe/templates/scanpipe/panels/file_tree_panel.html +++ b/scanpipe/templates/scanpipe/panels/codebase_tree_panel.html @@ -2,8 +2,8 @@ {% for node in children %}
  • {% if node.is_dir %} -
    -
    -
    -
    +
    +
    {% include "scanpipe/panels/codebase_tree_panel.html" with children=children path=path %}
    -
    -
    +
    +
    +
    {% include "scanpipe/panels/resource_table_panel.html" %}
    @@ -40,6 +99,7 @@ {% block scripts %} {% endblock %} \ No newline at end of file From e665feb904803502e2db46f5038126a7fa5e86ad Mon Sep 17 00:00:00 2001 From: Aayush Kumar Date: Tue, 19 Aug 2025 23:46:11 +0530 Subject: [PATCH 17/21] Revert filtering support and add reviewed changes Signed-off-by: Aayush Kumar --- scanpipe/models.py | 2 +- scanpipe/templates/scanpipe/panels/project_codebase.html | 8 +++++--- scanpipe/urls.py | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/scanpipe/models.py b/scanpipe/models.py index 126f85f550..a4c19020d4 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -232,7 +232,7 @@ def delete(self, *args, **kwargs): Note that projects with queued or running pipeline runs cannot be deleted. See the `_raise_if_run_in_progress` method. The following if statements should not be triggered unless the `.delete()` - method is directly call from a instance of this class. + method is directly call from an instance of this class. """ with suppress(redis.exceptions.ConnectionError, AttributeError): if self.status == self.Status.RUNNING: diff --git a/scanpipe/templates/scanpipe/panels/project_codebase.html b/scanpipe/templates/scanpipe/panels/project_codebase.html index 06c0ca5e77..c05a77aaa1 100644 --- a/scanpipe/templates/scanpipe/panels/project_codebase.html +++ b/scanpipe/templates/scanpipe/panels/project_codebase.html @@ -1,7 +1,9 @@ {% endif %} -{% else %} -
    -
    - -
    -

    - {% if path %} - No resources found in this directory. - {% else %} - Select a file or folder from the tree to view its contents. - {% endif %} -

    -
    -{% endif %}
    \ No newline at end of file From a51f906612eed4eeb4d82177d49b4847e4837519 Mon Sep 17 00:00:00 2001 From: Aayush Kumar Date: Thu, 28 Aug 2025 16:49:06 +0530 Subject: [PATCH 21/21] Use PrefetchRelatedViewMixin instead of doing it manually Signed-off-by: Aayush Kumar --- scanpipe/views.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scanpipe/views.py b/scanpipe/views.py index 6fddbe866c..b62eafe5f0 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -2800,16 +2800,22 @@ def get(self, request, *args, **kwargs): class CodebaseResourceTableView( ConditionalLoginRequired, + PrefetchRelatedViewMixin, ProjectRelatedViewMixin, TableColumnsMixin, PaginatedFilterView, ): + prefetch_related = [ + Prefetch( + "discovered_packages", + queryset=DiscoveredPackage.objects.only_package_url_fields(), + ) + ] def get_filterset_kwargs(self, filterset_class): """Remove 'path' from filterset data when showing children of a directory.""" kwargs = super().get_filterset_kwargs(filterset_class) path = self.request.GET.get("path") if path: - # Only remove 'path' if we're showing children of a directory base_qs = super().get_queryset() if base_qs.filter(path=path, type="directory").exists(): data = kwargs.get("data") @@ -2861,7 +2867,6 @@ def get_filterset_kwargs(self, filterset_class): def get_queryset(self): path = self.request.GET.get("path") base_qs = super().get_queryset() - # Default: all resources for the project queryset = base_qs if path: dir_qs = base_qs.filter(path=path, type="directory") @@ -2886,7 +2891,6 @@ def get_queryset(self): "compliance_alert", "package_data", ) - .prefetch_related("discovered_packages") .order_by("type", "path") )