Skip to content

Commit 9db8b12

Browse files
pombredannepombredanne
authored
Merge pull request #955 from nexB/faster-vuln-search
Make search for vulnerabilities faster
2 parents 8eaa86a + 47fed93 commit 9db8b12

File tree

9 files changed

+147
-53
lines changed

9 files changed

+147
-53
lines changed

‎setup.cfg‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,8 @@ dev =
104104
ipython==8.0.1
105105
# used for testing
106106
commoncode
107+
# debug
108+
django-debug-toolbar
107109

108110
[options.entry_points]
109111
console_scripts =

‎vulnerabilities/api.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -256,7 +256,10 @@ def bulk_search(self, request):
256256

257257
@action(detail=False, methods=["get"])
258258
def all(self, request):
259-
vulnerable_packages = Package.objects.vulnerable().only(*PackageURL._fields)
259+
"""
260+
Return all the vulnerable Package URLs.
261+
"""
262+
vulnerable_packages = Package.objects.vulnerable().only(*PackageURL._fields).distinct()
260263
vulnerable_purls = [str(package.purl) for package in vulnerable_packages]
261264
return Response(vulnerable_purls)
262265

‎vulnerabilities/migrations/0028_alter_packagerelatedvulnerability_fix.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# Generated by Django 4.0.7 on 2022-10-19 16:18
2+
3+
from django.db import migrations, models
4+
5+
6+
class Migration(migrations.Migration):
7+
8+
dependencies = [
9+
('vulnerabilities', '0027_alter_vulnerabilityreference_url'),
10+
]
11+
12+
operations = [
13+
migrations.AlterField(
14+
model_name='packagerelatedvulnerability',
15+
name='fix',
16+
field=models.BooleanField(db_index=True, default=False, help_text='Does this relation fix the specified vulnerability ?'),
17+
),
18+
]

‎vulnerabilities/models.py‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
from django.core.validators import MaxValueValidator
1818
from django.core.validators import MinValueValidator
1919
from django.db import models
20+
from django.db.models import Count
21+
from django.db.models import Q
2022
from django.db.models.functions import Length
2123
from django.db.models.functions import Trim
2224
from django.dispatch import receiver
@@ -86,8 +88,7 @@ def __str__(self):
8688

8789
@property
8890
def severities(self):
89-
for reference in self.references.all():
90-
yield from VulnerabilitySeverity.objects.filter(reference=reference.id)
91+
return VulnerabilitySeverity.objects.filter(reference__in=self.references.all())
9192

9293
@property
9394
def vulnerable_to(self):
@@ -202,7 +203,19 @@ def vulnerable(self):
202203
"""
203204
Return all vulnerable packages.
204205
"""
205-
return Package.objects.filter(packagerelatedvulnerability__fix=False).distinct()
206+
return self.filter(packagerelatedvulnerability__fix=False)
207+
208+
def with_vulnerability_counts(self):
209+
return self.annotate(
210+
vulnerability_count=Count(
211+
"vulnerabilities",
212+
filter=Q(packagerelatedvulnerability__fix=False),
213+
),
214+
patched_vulnerability_count=Count(
215+
"vulnerabilities",
216+
filter=Q(packagerelatedvulnerability__fix=True),
217+
),
218+
)
206219

207220

208221
class Package(PackageURLMixin):
@@ -310,7 +323,9 @@ class PackageRelatedVulnerability(models.Model):
310323
)
311324

312325
fix = models.BooleanField(
313-
default=False, help_text="Does this relation fix the specified vulnerability ?"
326+
default=False,
327+
db_index=True,
328+
help_text="Does this relation fix the specified vulnerability ?",
314329
)
315330

316331
class Meta:

‎vulnerabilities/templates/vulnerability_details.html‎

Lines changed: 25 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -37,21 +37,21 @@
3737
<li data-tab="fixed-by">
3838
<a>
3939
<span>
40-
Fixed by packages ({{ vulnerability.resolved_to|length }})
40+
Fixed by packages ({{ resolved_to|length }})
4141
</span>
4242
</a>
4343
</li>
4444
<li data-tab="affected-packages">
4545
<a>
4646
<span>
47-
Affected packages ({{ vulnerability.vulnerable_to|length }})
47+
Affected packages ({{ vulnerable_to|length }})
4848
</span>
4949
</a>
5050
</li>
5151
<li data-tab="references">
5252
<a>
5353
<span>
54-
References ({{ vulnerability.references.all|length }})
54+
References ({{ references|length }})
5555
</span>
5656
</a>
5757
</li>
@@ -69,7 +69,7 @@
6969
<tr>
7070
<td class="two-col-left">Aliases</td>
7171
<td class="two-col-right">
72-
{% for alias in vulnerability.aliases.all %}
72+
{% for alias in aliases %}
7373
{% if alias.url %}
7474
<a href={{ alias.url }} target="_blank">{{ alias }}<i class="fa fa-external-link fa_link_custom"></i></a>
7575
{% else %}
@@ -121,11 +121,11 @@
121121

122122

123123
<div class="has-text-weight-bold tab-nested-div ml-1 mb-1 mt-6">
124-
Fixed by packages ({{ vulnerability.resolved_to.all|length }})
124+
Fixed by packages ({{ resolved_to|length }})
125125
</div>
126126
<div class="tab-nested-div">
127127
<table class="table is-bordered is-striped is-narrow is-hoverable is-fullwidth gray-header-border">
128-
{% for package in vulnerability.resolved_to.all|slice:":3" %}
128+
{% for package in resolved_to|slice:":3" %}
129129
<tr>
130130
<td>
131131
<a href="{{ package.get_absolute_url }}" target="_self">{{ package.purl }}</a>
@@ -139,7 +139,7 @@
139139
</td>
140140
</tr>
141141
{% endfor %}
142-
{% if vulnerability.resolved_to.all|length > 3 %}
142+
{% if resolved_to|length > 3 %}
143143
<tr>
144144
<td>
145145
... see <i>Fixed by packages</i> tab for more
@@ -150,11 +150,11 @@
150150
</div>
151151

152152
<div class="has-text-weight-bold tab-nested-div ml-1 mb-1 mt-6">
153-
Affected packages ({{ vulnerability.vulnerable_to.all|length }})
153+
Affected packages ({{ vulnerable_to|length }})
154154
</div>
155155
<div class="tab-nested-div">
156156
<table class="table is-bordered is-striped is-narrow is-hoverable is-fullwidth gray-header-border">
157-
{% for package in vulnerability.vulnerable_to.all|slice:":3" %}
157+
{% for package in vulnerable_to|slice:":3" %}
158158
<tr>
159159
<td>
160160
<a href="{{ package.get_absolute_url }}" target="_self">{{ package.purl }}</a>
@@ -168,7 +168,7 @@
168168
</td>
169169
</tr>
170170
{% endfor %}
171-
{% if vulnerability.vulnerable_to.all|length > 3 %}
171+
{% if vulnerable_to|length > 3 %}
172172
<tr>
173173
<td>
174174
... see <i>Affected packages</i> tab for more
@@ -187,7 +187,7 @@
187187
<th> URL </th>
188188
</tr>
189189
</thead>
190-
{% for ref in vulnerability.references.all %}
190+
{% for ref in references %}
191191
<tr>
192192
{% if ref.reference_id %}
193193
<td class="wrap-strings">{{ ref.reference_id }}</td>
@@ -210,23 +210,23 @@
210210
<table class="table is-bordered is-striped is-narrow is-hoverable is-fullwidth">
211211
<thead>
212212
<tr>
213-
<th><span class="has-tooltip-multiline has-tooltip-black has-tooltip-arrow has-tooltip-text-left" data-tooltip="The package url or purl is a URL string used to identify and locate a software package.">Package URL</span></th>
214-
<th style="width: 225px;"><span class="has-tooltip-multiline has-tooltip-black has-tooltip-arrow has-tooltip-text-left" data-tooltip="This is the number of vulnerabilities that affect the package.">Affected by vulnerabilities</span></th>
215-
<th style="width: 225px;"><span class="has-tooltip-multiline has-tooltip-black has-tooltip-arrow has-tooltip-text-left" data-tooltip="This is the number of vulnerabilities fixed by the package.">Fixing vulnerabilities</span></th>
213+
<th><span
214+
class="has-tooltip-multiline has-tooltip-black has-tooltip-arrow has-tooltip-text-left"
215+
data-tooltip="The package url or purl is a URL string used to identify and locate a software package.">
216+
Package URL</span>
217+
</th>
216218
</tr>
217219
</thead>
218220
<tbody>
219-
{% for package in vulnerability.vulnerable_to.all %}
221+
{% for package in vulnerable_to %}
220222
<tr>
221223
<td>
222224
<a href="{{ package.get_absolute_url }}?search={{ package.purl }}" target="_self">{{ package.purl }}</a>
223225
</td>
224-
<td>{{ package.vulnerable_to|length }}</td>
225-
<td>{{ package.resolved_to|length }}</td>
226226
</tr>
227227
{% empty %}
228228
<tr>
229-
<td colspan="3">
229+
<td>
230230
This vulnerability is not known to affect any packages.
231231
</td>
232232
</tr>
@@ -239,23 +239,23 @@
239239
<table class="table is-bordered is-striped is-narrow is-hoverable is-fullwidth">
240240
<thead>
241241
<tr>
242-
<th><span class="has-tooltip-multiline has-tooltip-black has-tooltip-arrow has-tooltip-text-left" data-tooltip="The package url or purl is a URL string used to identify and locate a software package.">Package URL</span></th>
243-
<th style="width: 225px;"><span class="has-tooltip-multiline has-tooltip-black has-tooltip-arrow has-tooltip-text-left" data-tooltip="This is the number of vulnerabilities that affect the package.">Affected by vulnerabilities</span></th>
244-
<th style="width: 225px;"><span class="has-tooltip-multiline has-tooltip-black has-tooltip-arrow has-tooltip-text-left" data-tooltip="This is the number of vulnerabilities fixed by the package.">Fixing vulnerabilities</span></th>
242+
<th><span
243+
class="has-tooltip-multiline has-tooltip-black has-tooltip-arrow has-tooltip-text-left"
244+
data-tooltip="The package url or purl is a URL string used to identify and locate a software package.">
245+
Package URL</span>
246+
</th>
245247
</tr>
246248
</thead>
247249
<tbody>
248-
{% for package in vulnerability.resolved_to.all %}
250+
{% for package in resolved_to %}
249251
<tr>
250252
<td>
251253
<a href="{{ package.get_absolute_url }}?search={{ package.purl }}" target="_self">{{ package.purl }}</a>
252254
</td>
253-
<td>{{ package.vulnerable_to|length }}</td>
254-
<td>{{ package.resolved_to|length }}</td>
255255
</tr>
256256
{% empty %}
257257
<tr>
258-
<td colspan="3">
258+
<td>
259259
This vulnerability is not known to be fixed by any packages.
260260
</td>
261261
</tr>

‎vulnerabilities/tests/test_fix_models.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,9 @@ def setUp(self):
5050

5151
def test_get_vulnerable_packages(self):
5252
vuln_packages = Package.objects.vulnerable()
53-
assert vuln_packages.count() == 10
54-
vuln_purls = [pkg.purl for pkg in vuln_packages.only(*PackageURL._fields)]
53+
assert vuln_packages.count() == 20
54+
assert vuln_packages.distinct().count() == 10
55+
vuln_purls = [pkg.purl for pkg in vuln_packages.distinct().only(*PackageURL._fields)]
5556
assert vuln_purls == [
5657
"pkg:generic/nginx/test@0",
5758
"pkg:generic/nginx/test@1",

‎vulnerabilities/views.py‎

Lines changed: 39 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -126,27 +126,34 @@ def get_context_data(self, **kwargs):
126126
def get_queryset(self, query=None):
127127
query = query or self.request.GET.get("search") or ""
128128
qs = self.model.objects
129+
query = query.strip()
129130
if not query:
130131
return qs.none()
131-
qs = (
132-
qs.filter(
133-
Q(vulnerability_id__icontains=query)
134-
| Q(aliases__alias__icontains=query)
135-
| Q(references__id__icontains=query)
136-
| Q(summary__icontains=query)
137-
)
138-
.order_by("vulnerability_id")
139-
.annotate(
140-
vulnerable_package_count=Count(
141-
"packages", filter=Q(packagerelatedvulnerability__fix=False), distinct=True
142-
),
143-
patched_package_count=Count(
144-
"packages", filter=Q(packagerelatedvulnerability__fix=True), distinct=True
145-
),
146-
)
147-
.prefetch_related()
132+
133+
# middle ground, exact on vulnerability_id
134+
qssearch = qs.filter(vulnerability_id=query)
135+
if not qssearch.exists():
136+
# middle ground, exact on alias
137+
qssearch = qs.filter(aliases__alias=query)
138+
if not qssearch.exists():
139+
# middle ground, slow enough
140+
qssearch = qs.filter(
141+
Q(vulnerability_id__icontains=query) | Q(aliases__alias__icontains=query)
142+
)
143+
if not qssearch.exists():
144+
# last resort super slow
145+
qssearch = qs.filter(
146+
Q(references__id__icontains=query) | Q(summary__icontains=query)
147+
)
148+
149+
return qssearch.order_by("vulnerability_id").annotate(
150+
vulnerable_package_count=Count(
151+
"packages", filter=Q(packagerelatedvulnerability__fix=False), distinct=True
152+
),
153+
patched_package_count=Count(
154+
"packages", filter=Q(packagerelatedvulnerability__fix=True), distinct=True
155+
),
148156
)
149-
return qs
150157

151158

152159
class PackageDetails(DetailView):
@@ -190,11 +197,22 @@ class VulnerabilityDetails(DetailView):
190197
slug_url_kwarg = "vulnerability_id"
191198
slug_field = "vulnerability_id"
192199

200+
def get_queryset(self):
201+
return super().get_queryset().prefetch_related("references", "aliases")
202+
193203
def get_context_data(self, **kwargs):
194204
context = super().get_context_data(**kwargs)
195-
context["vulnerability"] = self.object
196-
context["vulnerability_search_form"] = VulnerabilitySearchForm(self.request.GET)
197-
context["severities"] = list(self.object.severities)
205+
context.update(
206+
{
207+
"vulnerability": self.object,
208+
"vulnerability_search_form": VulnerabilitySearchForm(self.request.GET),
209+
"severities": list(self.object.severities),
210+
"references": self.object.references.all(),
211+
"aliases": self.object.aliases.all(),
212+
"resolved_to": self.object.resolved_to.all(),
213+
"vulnerable_to": self.object.vulnerable_to.all(),
214+
}
215+
)
198216
return context
199217

200218

‎vulnerablecode/settings.py‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@
3333
# SECURITY WARNING: don't run with debug turned on in production
3434
DEBUG = env.bool("VULNERABLECODE_DEBUG", default=False)
3535

36+
# SECURITY WARNING: don't run with debug turned on in production
37+
DEBUG_TOOLBAR = env.bool("VULNERABLECODE_DEBUG_TOOLBAR", default=False)
38+
3639
# SECURITY WARNING: don't run with debug turned on in production
3740
DEBUG_UI = env.bool("VULNERABLECODE_DEBUG_UI", default=False)
3841

@@ -57,6 +60,7 @@
5760
"widget_tweaks",
5861
)
5962

63+
6064
MIDDLEWARE = (
6165
"django.middleware.security.SecurityMiddleware",
6266
"django.contrib.sessions.middleware.SessionMiddleware",
@@ -187,3 +191,30 @@
187191

188192
if not VULNERABLECODEIO_REQUIRE_AUTHENTICATION:
189193
REST_FRAMEWORK["DEFAULT_PERMISSION_CLASSES"] = ("rest_framework.permissions.AllowAny",)
194+
195+
196+
if DEBUG_TOOLBAR:
197+
INSTALLED_APPS += ("debug_toolbar",)
198+
199+
MIDDLEWARE += ("debug_toolbar.middleware.DebugToolbarMiddleware",)
200+
201+
DEBUG_TOOLBAR_PANELS = (
202+
"debug_toolbar.panels.history.HistoryPanel",
203+
"debug_toolbar.panels.versions.VersionsPanel",
204+
"debug_toolbar.panels.timer.TimerPanel",
205+
"debug_toolbar.panels.settings.SettingsPanel",
206+
"debug_toolbar.panels.headers.HeadersPanel",
207+
"debug_toolbar.panels.request.RequestPanel",
208+
"debug_toolbar.panels.sql.SQLPanel",
209+
"debug_toolbar.panels.staticfiles.StaticFilesPanel",
210+
"debug_toolbar.panels.templates.TemplatesPanel",
211+
"debug_toolbar.panels.cache.CachePanel",
212+
"debug_toolbar.panels.signals.SignalsPanel",
213+
"debug_toolbar.panels.logging.LoggingPanel",
214+
"debug_toolbar.panels.redirects.RedirectsPanel",
215+
"debug_toolbar.panels.profiling.ProfilingPanel",
216+
)
217+
218+
INTERNAL_IPS = [
219+
"127.0.0.1",
220+
]

‎vulnerablecode/urls.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
from vulnerabilities.views import VulnerabilityDetails
2424
from vulnerabilities.views import VulnerabilitySearch
2525
from vulnerabilities.views import schema_view
26+
from vulnerablecode.settings import DEBUG_TOOLBAR
2627

2728

2829
# See the comment at https://stackoverflow.com/a/46163870.
@@ -54,3 +55,8 @@ def __init__(self, *args, **kwargs):
5455
# disabled for now
5556
# path("admin/", admin.site.urls),
5657
]
58+
59+
if DEBUG_TOOLBAR:
60+
urlpatterns += [
61+
path("__debug__/", include("debug_toolbar.urls")),
62+
]

0 commit comments

Comments
 (0)