Skip to content

Commit 469d20e

Browse files
TG1999TG1999
authored
Merge pull request #985 from nexB/969-migrate-postgresql-importer
Migrate postgresql importer
2 parents bf49673 + 1aca009 commit 469d20e

File tree

8 files changed

+288
-225
lines changed

8 files changed

+288
-225
lines changed

‎CHANGELOG.rst‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,12 @@ Release notes
33

44

55

6+
Version v30.3.2
7+
----------------
8+
9+
- We re-enabled support for the PostgreSQL securities advisories importer.
10+
11+
612
Version v30.3.1
713
----------------
814

@@ -17,10 +23,10 @@ Version v30.3.0
1723
This is a feature update release including minor bug fixes and the introduction
1824
of API keys and API throttling.
1925

20-
- We enabled API throttling for a basic user and for a staff user
26+
- We enabled API throttling for a basic user and for a staff user
2127
they can have unlimited access on API.
2228

23-
- We added throttle rate for each API endpoint and it can be
29+
- We added throttle rate for each API endpoint and it can be
2430
configured from the settings #991 https://github.com/nexB/vulnerablecode/issues/991
2531

2632
- We improved how we import NVD data

‎vulnerabilities/importers/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
from vulnerabilities.importers import nginx
1717
from vulnerabilities.importers import nvd
1818
from vulnerabilities.importers import openssl
19+
from vulnerabilities.importers import postgresql
1920
from vulnerabilities.importers import pypa
2021
from vulnerabilities.importers import pysec
2122
from vulnerabilities.importers import redhat
@@ -31,6 +32,7 @@
3132
pysec.PyPIImporter,
3233
debian.DebianImporter,
3334
gitlab.GitLabAPIImporter,
35+
postgresql.PostgreSQLImporter,
3436
pypa.PyPaImporter,
3537
archlinux.ArchlinuxImporter,
3638
ubuntu.UbuntuImporter,

‎vulnerabilities/importers/postgresql.py‎

Lines changed: 54 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -12,35 +12,40 @@
1212
import requests
1313
from bs4 import BeautifulSoup
1414
from packageurl import PackageURL
15+
from univers.version_range import GenericVersionRange
16+
from univers.versions import GenericVersion
1517

1618
from vulnerabilities import severity_systems
1719
from vulnerabilities.importer import AdvisoryData
20+
from vulnerabilities.importer import AffectedPackage
1821
from vulnerabilities.importer import Importer
1922
from vulnerabilities.importer import Reference
2023
from vulnerabilities.importer import VulnerabilitySeverity
21-
from vulnerabilities.utils import nearest_patched_package
2224

2325

2426
class PostgreSQLImporter(Importer):
2527

2628
root_url = "https://www.postgresql.org/support/security/"
29+
license_url = "https://www.postgresql.org/about/licence/"
30+
spdx_license_expression = "PostgreSQL"
2731

28-
def updated_advisories(self):
29-
advisories = []
32+
def advisory_data(self):
3033
known_urls = {self.root_url}
3134
visited_urls = set()
35+
data_by_url = {}
3236
while True:
3337
unvisited_urls = known_urls - visited_urls
3438
for url in unvisited_urls:
3539
data = requests.get(url).content
36-
advisories.extend(to_advisories(data))
40+
data_by_url[url] = data
3741
visited_urls.add(url)
3842
known_urls.update(find_advisory_urls(data))
3943

4044
if known_urls == visited_urls:
4145
break
4246

43-
return self.batch_advisories(advisories)
47+
for url, data in data_by_url.items():
48+
yield from to_advisories(data)
4449

4550

4651
def to_advisories(data):
@@ -54,29 +59,43 @@ def to_advisories(data):
5459
if "windows" in summary.lower():
5560
pkg_qualifiers = {"os": "windows"}
5661

57-
affected_packages = [
58-
PackageURL(
59-
type="generic",
60-
name="postgresql",
61-
version=version.strip(),
62-
qualifiers=pkg_qualifiers,
63-
)
64-
for version in affected_col.text.split(",")
65-
]
66-
67-
fixed_packages = [
68-
PackageURL(
69-
type="generic",
70-
name="postgresql",
71-
version=version.strip(),
72-
qualifiers=pkg_qualifiers,
62+
affected_packages = []
63+
affected_version_list = affected_col.text.split(",")
64+
fixed_version_list = fixed_col.text.split(",")
65+
66+
if fixed_version_list:
67+
for fixed_version in fixed_version_list:
68+
affected_packages.append(
69+
AffectedPackage(
70+
package=PackageURL(
71+
name="postgresql",
72+
# TODO: See https://github.com/nexB/vulnerablecode/issues/990
73+
type="generic",
74+
qualifiers=pkg_qualifiers,
75+
),
76+
affected_version_range=GenericVersionRange.from_versions(
77+
affected_version_list
78+
)
79+
if affected_version_list
80+
else None,
81+
fixed_version=GenericVersion(fixed_version) if fixed_version else None,
82+
)
83+
)
84+
elif affected_version_list:
85+
affected_packages.append(
86+
AffectedPackage(
87+
package=PackageURL(
88+
name="postgresql",
89+
# TODO: See https://github.com/nexB/vulnerablecode/issues/990
90+
type="generic",
91+
qualifiers=pkg_qualifiers,
92+
),
93+
affected_version_range=GenericVersionRange.from_versions(affected_version_list),
94+
)
7395
)
74-
for version in fixed_col.text.split(",")
75-
if version
76-
]
77-
96+
cve_id = ""
7897
try:
79-
cve_id = ref_col.select("nobr")[0].text
98+
cve_id = ref_col.select(".nobr")[0].text
8099
# This is for the anomaly in https://www.postgresql.org/support/security/8.1/ 's
81100
# last entry
82101
except IndexError:
@@ -102,21 +121,20 @@ def to_advisories(data):
102121
)
103122
severities.append(severity)
104123
references.append(Reference(url=link, severities=severities))
105-
106-
advisories.append(
107-
AdvisoryData(
108-
vulnerability_id=cve_id,
109-
summary=summary,
110-
references=references,
111-
affected_packages=nearest_patched_package(affected_packages, fixed_packages),
124+
if cve_id:
125+
advisories.append(
126+
AdvisoryData(
127+
aliases=[cve_id],
128+
summary=summary,
129+
references=references,
130+
affected_packages=affected_packages,
131+
)
112132
)
113-
)
114-
115133
return advisories
116134

117135

118136
def find_advisory_urls(page_data):
119-
soup = BeautifulSoup(page_data)
137+
soup = BeautifulSoup(page_data, features="lxml")
120138
return {
121139
urlparse.urljoin("https://www.postgresql.org/", a_tag.attrs["href"])
122140
for a_tag in soup.select("h3+ p a")

‎vulnerabilities/models.py‎

Lines changed: 33 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -317,14 +317,40 @@ class Meta:
317317
ordering = ["vulnerability", "reference"]
318318

319319

320+
def purl_to_dict(purl: PackageURL):
321+
"""
322+
Return a dict of purl components suitable for use in a queryset.
323+
We need to have specific empty values for using in querysets because of our peculiar model structure.
324+
325+
For example::
326+
>>> purl_to_dict(PackageURL.from_string("pkg:generic/postgres"))
327+
{'type': 'generic', 'namespace': '', 'name': 'postgres', 'version': '', 'qualifiers': {}, 'subpath': ''}
328+
>>> purl_to_dict(PackageURL.from_string("pkg:generic/postgres/[email protected]?foo=bar#baz"))
329+
{'type': 'generic', 'namespace': 'postgres', 'name': 'postgres', 'version': '1.2', 'qualifiers': {'foo': 'bar'}, 'subpath': 'baz'}
330+
"""
331+
if isinstance(purl, str):
332+
purl = PackageURL.from_string(purl)
333+
334+
return dict(
335+
type=purl.type,
336+
namespace=purl.namespace or "",
337+
name=purl.name,
338+
version=purl.version or "",
339+
qualifiers=purl.qualifiers or {},
340+
subpath=purl.subpath or "",
341+
)
342+
343+
320344
class PackageQuerySet(BaseQuerySet, PackageURLQuerySet):
321345
def get_or_create_from_purl(self, purl: PackageURL):
322346
"""
323347
Return an existing or new Package (created if neeed) given a
324348
``purl`` PackageURL.
325349
"""
326-
purl_fields = without_empty_values(purl.to_dict(encode=True))
327-
package, _ = Package.objects.get_or_create(**purl_fields)
350+
if isinstance(purl, str):
351+
purl = PackageURL.from_string(purl)
352+
353+
package, _ = Package.objects.get_or_create(**purl_to_dict(purl=purl))
328354
return package
329355

330356
def for_package_url_object(self, purl):
@@ -333,15 +359,12 @@ def for_package_url_object(self, purl):
333359
``purl`` string is validated and transformed into filtering lookups. If
334360
this is a PackageURL object it is reused as-is.
335361
"""
336-
if isinstance(purl, PackageURL):
337-
lookups = without_empty_values(purl.to_dict(encode=True))
338-
return self.filter(**lookups)
339-
340-
elif isinstance(purl, str):
341-
return self.for_package_url(purl, encode=False)
342-
343-
else:
362+
if not purl:
344363
return self.none()
364+
if isinstance(purl, str):
365+
purl = PackageURL.from_string(purl)
366+
lookups = without_empty_values(purl.to_dict(encode=True))
367+
return self.filter(**lookups)
345368

346369
def affected(self):
347370
"""

‎vulnerabilities/tests/conftest.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@ def no_rmtree(monkeypatch):
3737
"test_msr2019.py",
3838
"test_npm.py",
3939
"test_package_managers.py",
40-
"test_postgresql.py",
4140
"test_retiredotnet.py",
4241
"test_ruby.py",
4342
"test_rust.py",

‎vulnerabilities/tests/test_data/postgresql/advisories.html‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ <h2>Known security issues in all supported versions</h2>
109109

110110
<tr>
111111
<td>
112-
<nobr><a href="/support/security/CVE-2020-10733/">CVE-2020-10733</a></nobr><br>
112+
<a href="/support/security/CVE-2020-10733/" class="nobr">CVE-2020-10733</a><br>
113113
<a href="/about/news/postgresql-123-118-1013-9618-and-9522-released-2038/">Announcement</a><br>
114114
</td>
115115
<td>12, 11, 10, 9.6</td>
@@ -122,7 +122,7 @@ <h2>Known security issues in all supported versions</h2>
122122

123123
<tr>
124124
<td>
125-
<nobr><a href="/support/security/CVE-2020-1720/">CVE-2020-1720</a></nobr><br>
125+
<a href="/support/security/CVE-2020-1720/" class="nobr">CVE-2020-1720</a><br>
126126
<a href="/about/news/postgresql-122-117-1012-9617-9521-and-9426-released-2011/">Announcement</a><br>
127127
</td>
128128
<td>12, 11, 10, 9.6</td>

0 commit comments

Comments
 (0)