Skip to content

Commit 4b1aae7

Browse files
authored
Merge branch 'main' into add-curl-advisories-importer
2 parents fcd4c0c + 6727786 commit 4b1aae7

32 files changed

+10386
-145
lines changed

CHANGELOG.rst

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,12 @@
11
Release notes
22
=============
33

4+
Version v34.0.0rc4
5+
-------------------
6+
7+
- Drop migration for removing duplicated changelogs.
8+
9+
410
Version v34.0.0rc3
511
-------------------
612

docs/source/conf.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
"https://api.github.com/graphql", # Requires auth
3535
"https://anongit.gentoo.org/git/data/glsa.git", # Git only link
3636
"https://www.softwaretestinghelp.com/how-to-write-good-bug-report/", # Cloudflare protection
37+
"https://www.openssl.org/news/vulnerabilities.xml", # OpenSSL legacy advisory URL, not longer available
3738
]
3839

3940
# Add any Sphinx extension module names here, as strings. They can be

pyproject.toml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,6 @@ addopts = [
5959
"--ignore=vulnerabilities/importers/retiredotnet.py",
6060
"--ignore=vulnerabilities/importers/ruby.py",
6161
"--ignore=vulnerabilities/importers/rust.py",
62-
"--ignore=vulnerabilities/importers/safety_db.py",
6362
"--ignore=vulnerabilities/importers/suse_backports.py",
6463
"--ignore=vulnerabilities/importers/suse_scores.py",
6564
"--ignore=vulnerabilities/importers/ubuntu_usn.py",

requirements.txt

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ beautifulsoup4==4.10.0
1111
binaryornot==0.4.4
1212
black==22.3.0
1313
boolean.py==3.8
14-
certifi==2023.7.22
14+
certifi==2024.7.4
1515
cffi==1.15.0
1616
chardet==4.0.0
1717
charset-normalizer==2.0.12
@@ -36,15 +36,15 @@ freezegun==1.2.1
3636
frozenlist==1.3.0
3737
gitdb==4.0.9
3838
GitPython==3.1.41
39-
gunicorn==20.1.0
39+
gunicorn==22.0.0
4040
idna==3.3
4141
imagesize==1.3.0
4242
importlib-metadata==4.11.3
4343
iniconfig==1.1.1
4444
ipython==8.10.0
4545
isort==5.10.1
4646
jedi==0.18.1
47-
Jinja2==3.1.3
47+
Jinja2==3.1.4
4848
jsonschema==3.2.0
4949
license-expression==21.6.14
5050
lxml==4.9.1
@@ -81,7 +81,7 @@ python-dateutil==2.8.2
8181
python-dotenv==0.20.0
8282
pytz==2022.1
8383
PyYAML==6.0.1
84-
requests==2.31.0
84+
requests==2.32.0
8585
restructuredtext-lint==1.4.0
8686
saneyaml==0.6.0
8787
semantic-version==2.9.0
@@ -98,7 +98,7 @@ sphinxcontrib-htmlhelp==2.0.0
9898
sphinxcontrib-jsmath==1.0.1
9999
sphinxcontrib-qthelp==1.0.3
100100
sphinxcontrib-serializinghtml==1.1.5
101-
sqlparse==0.4.4
101+
sqlparse==0.5.0
102102
stack-data==0.2.0
103103
stevedore==3.5.0
104104
texttable==1.6.4
@@ -107,11 +107,11 @@ tomli==2.0.1
107107
traitlets==5.1.1
108108
typing_extensions==4.1.1
109109
univers==30.11.0
110-
urllib3==1.26.18
110+
urllib3==1.26.19
111111
wcwidth==0.2.5
112112
websocket-client==0.59.0
113113
yarl==1.7.2
114-
zipp==3.8.0
114+
zipp==3.19.1
115115
dateparser==1.1.1
116116
fetchcode==0.3.0
117117
cwe2==2.0.0

vulnerabilities/api.py

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
from rest_framework.throttling import UserRateThrottle
2626

2727
from vulnerabilities.models import Alias
28+
from vulnerabilities.models import Kev
2829
from vulnerabilities.models import Package
2930
from vulnerabilities.models import Vulnerability
3031
from vulnerabilities.models import VulnerabilityReference
@@ -167,6 +168,12 @@ def to_representation(self, instance):
167168
return representation
168169

169170

171+
class KEVSerializer(serializers.ModelSerializer):
172+
class Meta:
173+
model = Kev
174+
fields = ["date_added", "description", "required_action", "due_date", "resources_and_notes"]
175+
176+
170177
class VulnerabilitySerializer(BaseResourceSerializer):
171178
fixed_packages = MinimalPackageSerializer(
172179
many=True, source="filtered_fixed_packages", read_only=True
@@ -175,6 +182,7 @@ class VulnerabilitySerializer(BaseResourceSerializer):
175182

176183
references = VulnerabilityReferenceSerializer(many=True, source="vulnerabilityreference_set")
177184
aliases = AliasSerializer(many=True, source="alias")
185+
kev = KEVSerializer(read_only=True)
178186
weaknesses = WeaknessSerializer(many=True)
179187

180188
def to_representation(self, instance):
@@ -183,6 +191,10 @@ def to_representation(self, instance):
183191
weaknesses = data.get("weaknesses", [])
184192
data["weaknesses"] = [weakness for weakness in weaknesses if weakness is not None]
185193

194+
kev = data.get("kev", None)
195+
if not kev:
196+
data.pop("kev")
197+
186198
return data
187199

188200
class Meta:
@@ -196,6 +208,7 @@ class Meta:
196208
"affected_packages",
197209
"references",
198210
"weaknesses",
211+
"kev",
199212
]
200213

201214

vulnerabilities/importers/github.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
"PIP": "pypi",
3838
"RUBYGEMS": "gem",
3939
"NPM": "npm",
40+
"RUST": "cargo",
4041
# "GO": "golang",
4142
}
4243

@@ -47,7 +48,7 @@
4748
# TODO: We will try to gather more info from GH API
4849
# Check https://github.com/nexB/vulnerablecode/issues/1039#issuecomment-1366458885
4950
# Check https://github.com/nexB/vulnerablecode/issues/645
50-
# set of all possible values of first '%s' = {'MAVEN','COMPOSER', 'NUGET', 'RUBYGEMS', 'PYPI', 'NPM'}
51+
# set of all possible values of first '%s' = {'MAVEN','COMPOSER', 'NUGET', 'RUBYGEMS', 'PYPI', 'NPM', 'RUST'}
5152
# second '%s' is interesting, it will have the value '' for the first request,
5253
GRAPHQL_QUERY_TEMPLATE = """
5354
query{
@@ -139,7 +140,7 @@ def get_purl(pkg_type: str, github_name: str) -> Optional[PackageURL]:
139140
vendor, _, name = github_name.partition("/")
140141
return PackageURL(type=pkg_type, namespace=vendor, name=name)
141142

142-
if pkg_type in ("nuget", "pypi", "gem", "golang", "npm"):
143+
if pkg_type in ("nuget", "pypi", "gem", "golang", "npm", "cargo"):
143144
return PackageURL(type=pkg_type, name=github_name)
144145

145146
logger.error(f"get_purl: Unknown package type {pkg_type}")

vulnerabilities/improvers/__init__.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
#
99

1010
from vulnerabilities.improvers import valid_versions
11+
from vulnerabilities.improvers import vulnerability_kev
1112
from vulnerabilities.improvers import vulnerability_status
1213

1314
IMPROVERS_REGISTRY = [
@@ -28,6 +29,7 @@
2829
valid_versions.GithubOSVImprover,
2930
vulnerability_status.VulnerabilityStatusImprover,
3031
valid_versions.CurlImprover,
32+
vulnerability_kev.VulnerabilityKevImprover,
3133
]
3234

3335
IMPROVERS_REGISTRY = {x.qualified_name: x for x in IMPROVERS_REGISTRY}
Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
import logging
2+
from typing import Iterable
3+
4+
from django.db.models import QuerySet
5+
from sphinx.util import requests
6+
7+
from vulnerabilities.improver import Improver
8+
from vulnerabilities.improver import Inference
9+
from vulnerabilities.models import Advisory
10+
from vulnerabilities.models import Alias
11+
from vulnerabilities.models import Kev
12+
13+
logger = logging.getLogger(__name__)
14+
15+
16+
class VulnerabilityKevImprover(Improver):
17+
"""
18+
Known Exploited Vulnerabilities Improver
19+
"""
20+
21+
@property
22+
def interesting_advisories(self) -> QuerySet:
23+
# TODO Modify KEV improver to iterate over the vulnerabilities alias, not the advisory
24+
return [Advisory.objects.first()]
25+
26+
def get_inferences(self, advisory_data) -> Iterable[Inference]:
27+
"""
28+
Fetch Kev data, iterate over it to find the vulnerability with the specified alias, and create or update
29+
the Kev instance accordingly.
30+
"""
31+
32+
kev_url = (
33+
"https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json"
34+
)
35+
response = requests.get(kev_url)
36+
kev_data = response.json()
37+
if response.status_code != 200:
38+
logger.error(
39+
f"Failed to fetch the CISA Catalog of Known Exploited Vulnerabilities: {kev_url}"
40+
)
41+
return []
42+
43+
for kev_vul in kev_data.get("vulnerabilities", []):
44+
alias = Alias.objects.get_or_none(alias=kev_vul["cveID"])
45+
if not alias:
46+
continue
47+
48+
vul = alias.vulnerability
49+
50+
if not vul:
51+
continue
52+
53+
Kev.objects.update_or_create(
54+
vulnerability=vul,
55+
defaults={
56+
"description": kev_vul["shortDescription"],
57+
"date_added": kev_vul["dateAdded"],
58+
"required_action": kev_vul["requiredAction"],
59+
"due_date": kev_vul["dueDate"],
60+
"resources_and_notes": kev_vul["notes"],
61+
"known_ransomware_campaign_use": True
62+
if kev_vul["knownRansomwareCampaignUse"] == "Known"
63+
else False,
64+
},
65+
)
66+
return []

vulnerabilities/migrations/0057_alter_packagechangelog_software_version_and_more.py renamed to vulnerabilities/migrations/0055_alter_packagechangelog_software_version_and_more.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
1-
# Generated by Django 4.1.13 on 2024-02-26 13:52
1+
# Generated by Django 4.1.13 on 2024-03-18 08:35
22

33
from django.db import migrations, models
44

55

66
class Migration(migrations.Migration):
77

88
dependencies = [
9-
("vulnerabilities", "0056_alter_packagechangelog_unique_together_and_more"),
9+
("vulnerabilities", "0054_alter_packagechangelog_software_version_and_more"),
1010
]
1111

1212
operations = [

vulnerabilities/migrations/0055_remove_changelogs_with_same_data_different_software_version.py

Lines changed: 0 additions & 52 deletions
This file was deleted.

0 commit comments

Comments
 (0)