Skip to content

Commit bf49673

Browse files
pombredannepombredanne
authored
Merge pull request #1009 from nexB/vulntotal
Add Vulntotal
2 parents e38eb1b + 6d838e6 commit bf49673

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

56 files changed

+7600
-1
lines changed

‎setup.cfg‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,4 +116,4 @@ dev =
116116
[options.entry_points]
117117
console_scripts =
118118
vulnerablecode = vulnerablecode:command_line
119-
119+
vulntotal = vulntotal.vulntotal_cli:handler

‎vulnerabilities/tests/util_tests.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,17 @@ def check_results_against_json(
4242
with open(expected_file) as exp:
4343
expected = json.load(exp)
4444

45+
check_results_against_expected(results, expected)
46+
47+
48+
def check_results_against_expected(
49+
results,
50+
expected,
51+
):
52+
"""
53+
Check the JSON-serializable mapping or sequence ``results`` against the
54+
``expected``.
55+
"""
4556
# NOTE we redump the JSON as a YAML string for easier display of
4657
# the failures comparison/diff
4758
if results != expected:

‎vulntotal/__init__.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#
2+
# Copyright (c) nexB Inc. and others. All rights reserved.
3+
# VulnerableCode is a trademark of nexB Inc.
4+
# SPDX-License-Identifier: Apache-2.0
5+
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
6+
# See https://github.com/nexB/vulnerablecode for support or download.
7+
# See https://aboutcode.org for more information about nexB OSS projects.
8+
#

‎vulntotal/datasources/__init__.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#
2+
# Copyright (c) nexB Inc. and others. All rights reserved.
3+
# VulnerableCode is a trademark of nexB Inc.
4+
# SPDX-License-Identifier: Apache-2.0
5+
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
6+
# See https://github.com/nexB/vulnerablecode for support or download.
7+
# See https://aboutcode.org for more information about nexB OSS projects.
8+
#
9+
10+
from vulntotal.datasources import deps
11+
from vulntotal.datasources import github
12+
from vulntotal.datasources import gitlab
13+
from vulntotal.datasources import oss_index
14+
from vulntotal.datasources import osv
15+
from vulntotal.datasources import snyk
16+
from vulntotal.datasources import vulnerablecode
17+
from vulntotal.validator import DataSource
18+
19+
DATASOURCE_REGISTRY = {
20+
"deps": deps.DepsDataSource,
21+
"github": github.GithubDataSource,
22+
"gitlab": gitlab.GitlabDataSource,
23+
"oss_index": oss_index.OSSDataSource,
24+
"osv": osv.OSVDataSource,
25+
"snyk": snyk.SnykDataSource,
26+
"vulnerablecode": vulnerablecode.VulnerableCodeDataSource,
27+
}

‎vulntotal/datasources/deps.py‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
#
2+
# Copyright (c) nexB Inc. and others. All rights reserved.
3+
# VulnerableCode is a trademark of nexB Inc.
4+
# SPDX-License-Identifier: Apache-2.0
5+
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
6+
# See https://github.com/nexB/vulnerablecode for support or download.
7+
# See https://aboutcode.org for more information about nexB OSS projects.
8+
#
9+
10+
import logging
11+
from typing import Iterable
12+
from urllib.parse import quote
13+
14+
import requests
15+
16+
from vulntotal.validator import DataSource
17+
from vulntotal.validator import VendorData
18+
19+
logger = logging.getLogger(__name__)
20+
21+
22+
class DepsDataSource(DataSource):
23+
spdx_license_expression = "TODO"
24+
license_url = "TODO"
25+
26+
def fetch_json_response(self, url):
27+
response = requests.get(url)
28+
if not response.status_code == 200 or response.text == "Not Found":
29+
logger.error(f"Error while fetching {url}")
30+
return
31+
return response.json()
32+
33+
def datasource_advisory(self, purl) -> Iterable[VendorData]:
34+
payload = generate_meta_payload(purl)
35+
response = self.fetch_json_response(payload)
36+
if response:
37+
advisories = parse_advisories_from_meta(response)
38+
if advisories:
39+
for advisory in advisories:
40+
advisory_payload = generate_advisory_payload(advisory)
41+
fetched_advisory = self.fetch_json_response(advisory_payload)
42+
self._raw_dump.append(fetched_advisory)
43+
if fetched_advisory:
44+
return parse_advisory(fetched_advisory)
45+
46+
@classmethod
47+
def supported_ecosystem(cls):
48+
return {
49+
"npm": "npm",
50+
"maven": "maven",
51+
"golang": "go",
52+
"pypi": "pypi",
53+
"cargo": "cargo",
54+
# Coming soon
55+
# "nuget": "nuget",
56+
}
57+
58+
59+
def parse_advisory(advisory) -> Iterable[VendorData]:
60+
package = advisory["packages"][0]
61+
affected_versions = [event["version"] for event in package["versionsAffected"]]
62+
fixed_versions = [event["version"] for event in package["versionsUnaffected"]]
63+
yield VendorData(
64+
aliases=sorted(set(advisory["aliases"])),
65+
affected_versions=sorted(set(affected_versions)),
66+
fixed_versions=sorted(set(fixed_versions)),
67+
)
68+
69+
70+
def parse_advisories_from_meta(advisories_metadata):
71+
advisories = []
72+
dependencies = advisories_metadata.get("dependencies") or []
73+
for dependency in dependencies:
74+
advs = dependency.get("advisories") or []
75+
advisories.extend(advs)
76+
return advisories
77+
78+
79+
def generate_advisory_payload(advisory_meta):
80+
url_advisory = "https://deps.dev/_/advisory/{source}/{sourceID}"
81+
return url_advisory.format(source=advisory_meta["source"], sourceID=advisory_meta["sourceID"])
82+
83+
84+
def generate_meta_payload(purl):
85+
url_advisories_meta = "https://deps.dev/_/s/{ecosystem}/p/{package}/v/{version}/dependencies"
86+
supported_ecosystem = DepsDataSource.supported_ecosystem()
87+
if purl.type in supported_ecosystem:
88+
purl_version = purl.version
89+
purl_name = purl.name
90+
91+
if purl.type == "maven":
92+
if not purl.namespace:
93+
logger.error(f"Invalid Maven PURL {str(purl)}")
94+
return
95+
purl_name = quote(f"{purl.namespace}:{purl.name}", safe="")
96+
97+
elif purl.type == "golang":
98+
if purl.namespace:
99+
purl_name = quote(f"{purl.namespace}/{purl.name}", safe="")
100+
if not purl_version.startswith("v"):
101+
purl_version = f"v{purl_version}"
102+
103+
return url_advisories_meta.format(
104+
ecosystem=supported_ecosystem[purl.type],
105+
package=purl_name,
106+
version=purl_version,
107+
)

‎vulntotal/datasources/github.py‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
#
2+
# Copyright (c) nexB Inc. and others. All rights reserved.
3+
# VulnerableCode is a trademark of nexB Inc.
4+
# SPDX-License-Identifier: Apache-2.0
5+
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
6+
# See https://github.com/nexB/vulnerablecode for support or download.
7+
# See https://aboutcode.org for more information about nexB OSS projects.
8+
#
9+
10+
import logging
11+
from typing import Iterable
12+
13+
from vulnerabilities import utils
14+
from vulntotal.validator import DataSource
15+
from vulntotal.validator import VendorData
16+
from vulntotal.vulntotal_utils import github_constraints_satisfied
17+
18+
logger = logging.getLogger(__name__)
19+
20+
21+
class GithubDataSource(DataSource):
22+
spdx_license_expression = "TODO"
23+
license_url = "TODO"
24+
25+
def fetch_github(self, graphql_query):
26+
return utils.fetch_github_graphql_query(graphql_query)
27+
28+
def datasource_advisory(self, purl) -> Iterable[VendorData]:
29+
end_cursor = ""
30+
interesting_edges = []
31+
while True:
32+
queryset = generate_graphql_payload(purl, end_cursor)
33+
response = self.fetch_github(queryset)
34+
self._raw_dump.append(response)
35+
security_advisories = response["data"]["securityVulnerabilities"]
36+
interesting_edges.extend(extract_interesting_edge(security_advisories["edges"], purl))
37+
end_cursor = security_advisories["pageInfo"]["endCursor"]
38+
if not security_advisories["pageInfo"]["hasNextPage"]:
39+
break
40+
return parse_advisory(interesting_edges)
41+
42+
@classmethod
43+
def supported_ecosystem(cls):
44+
return {
45+
"maven": "MAVEN",
46+
"nuget": "NUGET",
47+
"composer": "COMPOSER",
48+
"pypi": "PIP",
49+
"gem": "RUBYGEMS",
50+
"golang": "GO",
51+
"rust": "RUST",
52+
"npm": "NPM",
53+
"erlang": "ERLANG",
54+
}
55+
56+
57+
def parse_advisory(interesting_edges) -> Iterable[VendorData]:
58+
for edge in interesting_edges:
59+
node = edge["node"]
60+
aliases = [aliase["value"] for aliase in node["advisory"]["identifiers"]]
61+
affected_versions = node["vulnerableVersionRange"].strip().replace(" ", "").split(",")
62+
fixed_versions = [node["firstPatchedVersion"]["identifier"]]
63+
yield VendorData(
64+
aliases=sorted(list(set(aliases))),
65+
affected_versions=sorted(list(set(affected_versions))),
66+
fixed_versions=sorted(list(set(fixed_versions))),
67+
)
68+
69+
70+
def extract_interesting_edge(edges, purl):
71+
interesting_edges = []
72+
for edge in edges:
73+
if github_constraints_satisfied(edge["node"]["vulnerableVersionRange"], purl.version):
74+
interesting_edges.append(edge)
75+
return interesting_edges
76+
77+
78+
def generate_graphql_payload(purl, end_cursor):
79+
GRAPHQL_QUERY_TEMPLATE = """
80+
query{
81+
securityVulnerabilities(first: 100, ecosystem: %s, package: "%s", %s){
82+
edges {
83+
node {
84+
advisory {
85+
identifiers {
86+
type
87+
value
88+
}
89+
summary
90+
references {
91+
url
92+
}
93+
severity
94+
publishedAt
95+
}
96+
firstPatchedVersion{
97+
identifier
98+
}
99+
package {
100+
name
101+
}
102+
vulnerableVersionRange
103+
}
104+
}
105+
pageInfo {
106+
hasNextPage
107+
endCursor
108+
}
109+
}
110+
}
111+
"""
112+
113+
supported_ecosystem = GithubDataSource.supported_ecosystem()
114+
115+
if purl.type not in supported_ecosystem:
116+
return
117+
118+
end_cursor_exp = ""
119+
ecosystem = supported_ecosystem[purl.type]
120+
package_name = purl.name
121+
122+
if end_cursor:
123+
end_cursor_exp = f'after: "{end_cursor}"'
124+
125+
if purl.type == "maven":
126+
if not purl.namespace:
127+
logger.error(f"Invalid Maven PURL {str(purl)}")
128+
return
129+
package_name = f"{purl.namespace}:{purl.name}"
130+
131+
elif purl.type == "composer":
132+
if not purl.namespace:
133+
logger.error(f"Invalid Composer PURL {str(purl)}")
134+
return
135+
package_name = f"{purl.namespace}/{purl.name}"
136+
137+
elif purl.type == "golang" and purl.namespace:
138+
package_name = f"{purl.namespace}/{purl.name}"
139+
140+
return {"query": GRAPHQL_QUERY_TEMPLATE % (ecosystem, package_name, end_cursor_exp)}

0 commit comments

Comments
 (0)