|
6 | 6 | # See https://github.com/nexB/vulnerablecode for support or download. |
7 | 7 | # See https://aboutcode.org for more information about nexB OSS projects. |
8 | 8 | # |
9 | | -import asyncio |
10 | 9 | import re |
| 10 | +from pathlib import Path |
11 | 11 | from typing import Set |
12 | 12 |
|
13 | 13 | import pytz |
14 | 14 | import saneyaml |
15 | 15 | from dateutil import parser |
16 | 16 | from packageurl import PackageURL |
17 | | -from univers.version_range import VersionRange |
| 17 | +from univers.version_constraint import VersionConstraint |
| 18 | +from univers.version_range import GitHubVersionRange |
| 19 | +from univers.version_range import GolangVersionRange |
18 | 20 | from univers.versions import SemverVersion |
19 | 21 |
|
20 | 22 | from vulnerabilities.importer import AdvisoryData |
21 | | -from vulnerabilities.importer import GitImporter |
| 23 | +from vulnerabilities.importer import AffectedPackage |
| 24 | +from vulnerabilities.importer import Importer |
22 | 25 | from vulnerabilities.importer import Reference |
23 | | -from vulnerabilities.package_managers import GitHubTagsAPI |
24 | | -from vulnerabilities.utils import nearest_patched_package |
25 | 26 | from vulnerabilities.utils import split_markdown_front_matter |
26 | 27 |
|
27 | 28 | is_release = re.compile(r"^[\d.]+$", re.IGNORECASE).match |
28 | 29 |
|
29 | 30 |
|
30 | | -class IstioImporter(GitImporter): |
31 | | - def __enter__(self): |
32 | | - super(IstioImporter, self).__enter__() |
| 31 | +class IstioImporter(Importer): |
| 32 | + spdx_license_expression = "Apache-2.0" |
| 33 | + license_url = "https://github.com/istio/istio.io/blob/master/LICENSE" |
| 34 | + repo_url = "git+https://github.com/istio/istio.io/" |
33 | 35 |
|
34 | | - if not getattr(self, "_added_files", None): |
35 | | - self._added_files, self._updated_files = self.file_changes( |
36 | | - recursive=True, file_ext="md", subdir="./content/en/news/security" |
37 | | - ) |
38 | | - self.version_api = GitHubTagsAPI() |
39 | | - self.set_api() |
40 | | - |
41 | | - def set_api(self): |
42 | | - asyncio.run(self.version_api.load_api(["istio/istio"])) |
43 | | - |
44 | | - def updated_advisories(self) -> Set[AdvisoryData]: |
45 | | - files = self._added_files.union(self._updated_files) |
46 | | - advisories = [] |
47 | | - for f in files: |
| 36 | + def advisory_data(self) -> Set[AdvisoryData]: |
| 37 | + self.clone(self.repo_url) |
| 38 | + path = Path(self.vcs_response.dest_dir) |
| 39 | + vuln = path / "content/en/news/security/" |
| 40 | + for file in vuln.glob("**/*.md"): |
48 | 41 | # Istio website has files with name starting with underscore, these contain metadata |
49 | 42 | # required for rendering the website. We're not interested in these. |
50 | 43 | # See also https://github.com/nexB/vulnerablecode/issues/563 |
51 | | - if f.endswith("_index.md"): |
| 44 | + file = str(file) |
| 45 | + if file.endswith("_index.md"): |
52 | 46 | continue |
53 | | - processed_data = self.process_file(f) |
54 | | - if processed_data: |
55 | | - advisories.extend(processed_data) |
56 | | - return self.batch_advisories(advisories) |
57 | | - |
58 | | - def get_pkg_versions_from_ranges(self, version_range_list, release_date): |
59 | | - """Takes a list of version ranges(affected) of a package |
60 | | - as parameter and returns a tuple of safe package versions and |
61 | | - vulnerable package versions""" |
62 | | - all_version = self.version_api.get("istio/istio", release_date).valid_versions |
63 | | - safe_pkg_versions = [] |
64 | | - vuln_pkg_versions = [] |
65 | | - version_ranges = [ |
66 | | - VersionRange.from_scheme_version_spec_string("semver", r) for r in version_range_list |
67 | | - ] |
68 | | - for version in all_version: |
69 | | - version_obj = SemverVersion(version) |
70 | | - if any([version_obj in v for v in version_ranges]): |
71 | | - vuln_pkg_versions.append(version) |
72 | | - |
73 | | - safe_pkg_versions = set(all_version) - set(vuln_pkg_versions) |
74 | | - return safe_pkg_versions, vuln_pkg_versions |
| 47 | + yield from self.process_file(file) |
75 | 48 |
|
76 | 49 | def process_file(self, path): |
77 | 50 |
|
78 | | - advisories = [] |
79 | | - |
80 | 51 | data = self.get_data_from_md(path) |
81 | | - release_date = parser.parse(data["publishdate"]).replace(tzinfo=pytz.UTC) |
82 | | - |
83 | | - releases = [] |
84 | | - if data.get("releases"): |
85 | | - for release in data["releases"]: |
86 | | - # If it is of form "All releases prior to x" |
87 | | - if "All releases prior" in release: |
88 | | - release = release.strip() |
89 | | - release = release.split(" ") |
90 | | - releases.append("<" + release[4]) |
91 | | - |
92 | | - # Eg. 'All releases 1.5 and later' |
93 | | - elif "All releases" in release and "and later" in release: |
94 | | - release = release.split()[2].strip() |
95 | | - releases.append(f">={release}") |
96 | | - |
97 | | - elif "to" in release: |
98 | | - release = release.strip() |
99 | | - release = release.split(" ") |
100 | | - lbound = ">=" + release[0] |
101 | | - ubound = "<=" + release[2] |
102 | | - releases.append(lbound + "," + ubound) |
103 | | - # If it is a single release |
104 | | - elif is_release(release): |
105 | | - releases.append(release) |
106 | | - |
107 | | - data["release_ranges"] = releases |
108 | | - |
109 | | - if not data.get("cves"): |
110 | | - data["cves"] = [""] |
111 | | - |
112 | | - for cve_id in data["cves"]: |
| 52 | + published_date = data.get("publishdate") |
| 53 | + release_date = None |
| 54 | + if published_date: |
| 55 | + release_date = parser.parse(published_date).replace(tzinfo=pytz.UTC) |
| 56 | + |
| 57 | + constraints = [] |
| 58 | + |
| 59 | + for release in data.get("releases") or []: |
| 60 | + # If it is of form "All releases prior to x" |
| 61 | + if "All releases prior" in release: |
| 62 | + _, _, release = release.strip().rpartition(" ") |
| 63 | + constraints.append( |
| 64 | + VersionConstraint(version=SemverVersion(release), comparator="<") |
| 65 | + ) |
113 | 66 |
|
114 | | - if not cve_id.startswith("CVE"): |
115 | | - cve_id = "" |
| 67 | + # Eg. 'All releases 1.5 and later' |
| 68 | + elif "All releases" in release and "and later" in release: |
| 69 | + # remove All releases from string |
| 70 | + release = release.replace("All releases", "").strip() |
| 71 | + # remove and later from string |
| 72 | + release = release.replace("and later", "").strip() |
| 73 | + if not is_release(release): |
| 74 | + continue |
| 75 | + constraints.append( |
| 76 | + VersionConstraint(version=SemverVersion(release), comparator=">=") |
| 77 | + ) |
116 | 78 |
|
117 | | - safe_pkg_versions = [] |
118 | | - vuln_pkg_versions = [] |
| 79 | + # Eg. 1.5 to 2.0 |
| 80 | + elif "to" in release: |
| 81 | + lower, _, upper = release.strip().partition("to") |
| 82 | + constraints.append(VersionConstraint(version=SemverVersion(lower), comparator=">=")) |
| 83 | + constraints.append(VersionConstraint(version=SemverVersion(upper), comparator="<=")) |
119 | 84 |
|
120 | | - if not data.get("release_ranges"): |
121 | | - data["release_ranges"] = [] |
| 85 | + # If it is a single release |
| 86 | + elif is_release(release): |
| 87 | + constraints.append( |
| 88 | + VersionConstraint(version=SemverVersion(release), comparator="=") |
| 89 | + ) |
122 | 90 |
|
123 | | - safe_pkg_versions, vuln_pkg_versions = self.get_pkg_versions_from_ranges( |
124 | | - data["release_ranges"], release_date |
125 | | - ) |
| 91 | + for cve_id in data.get("cves") or []: |
| 92 | + |
| 93 | + if not cve_id.startswith("CVE"): |
| 94 | + continue |
126 | 95 |
|
127 | 96 | affected_packages = [] |
128 | 97 |
|
129 | | - safe_purls_golang = [ |
130 | | - PackageURL(type="golang", name="istio", version=version) |
131 | | - for version in safe_pkg_versions |
132 | | - ] |
133 | | - |
134 | | - vuln_purls_golang = [ |
135 | | - PackageURL(type="golang", name="istio", version=version) |
136 | | - for version in vuln_pkg_versions |
137 | | - ] |
138 | | - |
139 | | - affected_packages.extend(nearest_patched_package(vuln_purls_golang, safe_purls_golang)) |
140 | | - |
141 | | - safe_purls_github = [ |
142 | | - PackageURL(type="github", name="istio", version=version) |
143 | | - for version in safe_pkg_versions |
144 | | - ] |
145 | | - |
146 | | - vuln_purls_github = [ |
147 | | - PackageURL(type="github", name="istio", version=version) |
148 | | - for version in vuln_pkg_versions |
149 | | - ] |
150 | | - |
151 | | - affected_packages.extend(nearest_patched_package(vuln_purls_github, safe_purls_github)) |
152 | | - |
153 | | - advisories.append( |
154 | | - AdvisoryData( |
155 | | - vulnerability_id=cve_id, |
156 | | - summary=data["description"], |
157 | | - affected_packages=affected_packages, |
158 | | - references=[ |
159 | | - Reference( |
160 | | - reference_id=data["title"], |
161 | | - url=f"https://istio.io/latest/news/security/{data['title']}/", |
162 | | - ) |
163 | | - ], |
| 98 | + if constraints: |
| 99 | + affected_packages.append( |
| 100 | + AffectedPackage( |
| 101 | + package=PackageURL(type="golang", namespace="istio.io", name="istio"), |
| 102 | + affected_version_range=GolangVersionRange(constraints=constraints), |
| 103 | + ) |
| 104 | + ) |
| 105 | + |
| 106 | + affected_packages.append( |
| 107 | + AffectedPackage( |
| 108 | + package=PackageURL(type="github", namespace="istio", name="istio"), |
| 109 | + affected_version_range=GitHubVersionRange(constraints=constraints), |
| 110 | + ) |
164 | 111 | ) |
165 | | - ) |
166 | 112 |
|
167 | | - return advisories |
| 113 | + title = data.get("title") or "" |
| 114 | + references = [] |
| 115 | + if title: |
| 116 | + references.append( |
| 117 | + Reference( |
| 118 | + reference_id=title, |
| 119 | + url=f"https://istio.io/latest/news/security/{title}/", |
| 120 | + ) |
| 121 | + ) |
| 122 | + |
| 123 | + summary = data.get("description") or "" |
| 124 | + |
| 125 | + yield AdvisoryData( |
| 126 | + aliases=[cve_id], |
| 127 | + summary=summary, |
| 128 | + affected_packages=affected_packages, |
| 129 | + references=references, |
| 130 | + date_published=release_date, |
| 131 | + ) |
168 | 132 |
|
169 | 133 | def get_data_from_md(self, path): |
170 | 134 | """Return a mapping of vulnerability data extracted from an advisory.""" |
|
0 commit comments