|
| 1 | +# |
| 2 | +# Copyright (c) nexB Inc. and others. All rights reserved. |
| 3 | +# http://nexb.com and https://github.com/nexB/vulnerablecode/ |
| 4 | +# The VulnTotal software is licensed under the Apache License version 2.0. |
| 5 | +# Data generated with VulnTotal require an acknowledgment. |
| 6 | +# |
| 7 | +# You may not use this software except in compliance with the License. |
| 8 | +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 |
| 9 | +# Unless required by applicable law or agreed to in writing, software distributed |
| 10 | +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR |
| 11 | +# CONDITIONS OF ANY KIND, either express or implied. See the License for the |
| 12 | +# specific language governing permissions and limitations under the License. |
| 13 | +# |
| 14 | +# When you publish or redistribute any data created with VulnTotal or any VulnTotal |
| 15 | +# derivative work, you must accompany this data with the following acknowledgment: |
| 16 | +# |
| 17 | +# Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES |
| 18 | +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from |
| 19 | +# VulnTotal should be considered or used as legal advice. Consult an Attorney |
| 20 | +# for any legal advice. |
| 21 | +# VulnTotal is a free software tool from nexB Inc. and others. |
| 22 | +# Visit https://github.com/nexB/vulnerablecode/ for support and download. |
| 23 | + |
| 24 | + |
| 25 | +import json |
| 26 | +import logging |
| 27 | +import os |
| 28 | +import shutil |
| 29 | +import tarfile |
| 30 | +from pathlib import Path |
| 31 | +from typing import Iterable |
| 32 | + |
| 33 | +import requests |
| 34 | +import saneyaml |
| 35 | +from fetchcode import fetch |
| 36 | +from packageurl import PackageURL |
| 37 | + |
| 38 | +from vulntotal.validator import DataSource |
| 39 | +from vulntotal.validator import VendorData |
| 40 | +from vulntotal.vulntotal_utils import gitlab_constraints_satisfied |
| 41 | + |
| 42 | +logger = logging.getLogger(__name__) |
| 43 | + |
| 44 | + |
| 45 | +class GitlabDataSource(DataSource): |
| 46 | + spdx_license_expression = "TODO" |
| 47 | + license_url = "TODO" |
| 48 | + |
| 49 | + def datasource_advisory(self, purl) -> Iterable[VendorData]: |
| 50 | + package_slug = get_package_slug(purl) |
| 51 | + location = download_subtree(package_slug, speculative_execution=True) |
| 52 | + if not location: |
| 53 | + clear_download(location) |
| 54 | + path = self.supported_ecosystem()[purl.type] |
| 55 | + casesensitive_package_slug = get_casesensitive_slug(path, package_slug) |
| 56 | + location = download_subtree(casesensitive_package_slug) |
| 57 | + if location: |
| 58 | + interesting_advisories = parse_interesting_advisories(location, purl.version, delete_download=True) |
| 59 | + return interesting_advisories |
| 60 | + clear_download(location) |
| 61 | + |
| 62 | + @classmethod |
| 63 | + def supported_ecosystem(cls): |
| 64 | + return { |
| 65 | + "composer": "packagist", |
| 66 | + "conan": "conan", |
| 67 | + "gem": "gem", |
| 68 | + "golang": "go", |
| 69 | + "maven": "maven", |
| 70 | + "npm": "npm", |
| 71 | + "nuget": "nuget", |
| 72 | + "pypi": "pypi", |
| 73 | + } |
| 74 | + |
| 75 | + |
| 76 | +def get_package_slug(purl): |
| 77 | + supported_ecosystem = GitlabDataSource.supported_ecosystem() |
| 78 | + |
| 79 | + if purl.type not in supported_ecosystem: |
| 80 | + return |
| 81 | + |
| 82 | + ecosystem = supported_ecosystem[purl.type] |
| 83 | + package_name = purl.name |
| 84 | + |
| 85 | + if purl.type in ("maven", "composer", "golang"): |
| 86 | + package_name = f"{purl.namespace}/{purl.name}" |
| 87 | + |
| 88 | + return f"{ecosystem}/{package_name}" |
| 89 | + |
| 90 | + |
| 91 | +def download_subtree(package_slug: str, speculative_execution=False): |
| 92 | + url = f"https://gitlab.com/gitlab-org/security-products/gemnasium-db/-/archive/master/gemnasium-db-master.tar.gz?path={package_slug}" |
| 93 | + response = fetch(url) |
| 94 | + if os.path.getsize(response.location) > 0: |
| 95 | + extracted_location = Path(response.location).parent.joinpath( |
| 96 | + "temp_vulntotal_gitlab_datasource" |
| 97 | + ) |
| 98 | + with tarfile.open(response.location, "r") as file_obj: |
| 99 | + file_obj.extractall(extracted_location) |
| 100 | + os.remove(response.location) |
| 101 | + return extracted_location |
| 102 | + if not speculative_execution: |
| 103 | + logger.error(f"{package_slug} doesn't exist") |
| 104 | + os.remove(response.location) |
| 105 | + |
| 106 | + |
| 107 | +def clear_download(location): |
| 108 | + if location: |
| 109 | + shutil.rmtree(location) |
| 110 | + |
| 111 | + |
| 112 | +def get_casesensitive_slug(path, package_slug): |
| 113 | + payload = [ |
| 114 | + { |
| 115 | + "operationName": "getPaginatedTree", |
| 116 | + "variables": { |
| 117 | + "projectPath": "gitlab-org/security-products/gemnasium-db", |
| 118 | + "ref": "master", |
| 119 | + "path": path, |
| 120 | + "nextPageCursor": "", |
| 121 | + "pageSize": 100, |
| 122 | + }, |
| 123 | + "query": """ |
| 124 | + fragment TreeEntry on Entry { |
| 125 | + flatPath |
| 126 | + } |
| 127 | + query getPaginatedTree($projectPath: ID!, $path: String, $ref: String!, $nextPageCursor: String) { |
| 128 | + project(fullPath: $projectPath) { |
| 129 | + repository { |
| 130 | + paginatedTree(path: $path, ref: $ref, after: $nextPageCursor) { |
| 131 | + pageInfo { |
| 132 | + endCursor |
| 133 | + startCursor |
| 134 | + hasNextPage |
| 135 | + } |
| 136 | + nodes { |
| 137 | + trees { |
| 138 | + nodes { |
| 139 | + ...TreeEntry |
| 140 | + } |
| 141 | + } |
| 142 | + } |
| 143 | + } |
| 144 | + } |
| 145 | + } |
| 146 | + } """, |
| 147 | + } |
| 148 | + ] |
| 149 | + url = "https://gitlab.com/api/graphql" |
| 150 | + hasnext = True |
| 151 | + |
| 152 | + while hasnext: |
| 153 | + response = requests.post(url, json=payload).json() |
| 154 | + paginated_tree = response[0]["data"]["project"]["repository"]["paginatedTree"] |
| 155 | + |
| 156 | + for slug in paginated_tree["nodes"][0]["trees"]["nodes"]: |
| 157 | + if slug["flatPath"].lower() == package_slug.lower(): |
| 158 | + return slug["flatPath"] |
| 159 | + |
| 160 | + # If the namespace/subfolder contains multiple packages, then progressive transverse through folders tree |
| 161 | + if package_slug.lower().startswith(slug["flatPath"].lower()): |
| 162 | + return get_gitlab_style_slug(slug["flatPath"], package_slug) |
| 163 | + |
| 164 | + payload[0]["variables"]["nextPageCursor"] = paginated_tree["pageInfo"]["endCursor"] |
| 165 | + hasnext = paginated_tree["pageInfo"]["hasNextPage"] |
| 166 | + |
| 167 | + |
| 168 | +def parse_interesting_advisories(location, version, delete_download=False) -> Iterable[VendorData]: |
| 169 | + path = Path(location) |
| 170 | + glob = "**/*.yml" |
| 171 | + files = (p for p in path.glob(glob) if p.is_file()) |
| 172 | + for file in sorted(files): |
| 173 | + with open(file) as f: |
| 174 | + gitlab_advisory = saneyaml.load(f) |
| 175 | + if gitlab_constraints_satisfied(gitlab_advisory["affected_range"], version): |
| 176 | + yield VendorData( |
| 177 | + aliases=gitlab_advisory["identifiers"], |
| 178 | + affected_versions=[gitlab_advisory["affected_range"]], |
| 179 | + fixed_versions=gitlab_advisory["fixed_versions"], |
| 180 | + ) |
| 181 | + if delete_download: |
| 182 | + clear_download(location) |
0 commit comments