diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index 82ee4525a..73ef6ef01 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -41,6 +41,7 @@ from vulnerabilities.pipelines import nvd_importer from vulnerabilities.pipelines import pypa_importer from vulnerabilities.pipelines import pysec_importer +from vulnerabilities.pipelines.v2_importers import aosp_importer from vulnerabilities.pipelines.v2_importers import apache_httpd_importer as apache_httpd_v2 from vulnerabilities.pipelines.v2_importers import archlinux_importer as archlinux_importer_v2 from vulnerabilities.pipelines.v2_importers import curl_importer as curl_importer_v2 @@ -81,6 +82,7 @@ mozilla_importer_v2.MozillaImporterPipeline, github_osv_importer_v2.GithubOSVImporterPipeline, redhat_importer_v2.RedHatImporterPipeline, + aosp_importer.AospImporterPipeline, nvd_importer.NVDImporterPipeline, github_importer.GitHubAPIImporterPipeline, gitlab_importer.GitLabImporterPipeline, diff --git a/vulnerabilities/pipelines/v2_importers/aosp_importer.py b/vulnerabilities/pipelines/v2_importers/aosp_importer.py new file mode 100644 index 000000000..391be0df7 --- /dev/null +++ b/vulnerabilities/pipelines/v2_importers/aosp_importer.py @@ -0,0 +1,115 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import json +import shutil +from datetime import timezone +from pathlib import Path + +import dateparser +from fetchcode.vcs import fetch_via_vcs + +from vulnerabilities.importer import AdvisoryData +from vulnerabilities.importer import ReferenceV2 +from vulnerabilities.importer import VulnerabilitySeverity +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 +from vulnerabilities.severity_systems import GENERIC + + +class AospImporterPipeline(VulnerableCodeBaseImporterPipelineV2): + """ + Pipeline to collect fix commits from Aosp Dataset: + """ + + pipeline_id = "aosp_dataset_fix_commits" + spdx_license_expression = "Apache-2.0" + license_url = "https://github.com/quarkslab/aosp_dataset/blob/master/LICENSE" + importer_name = "aosp_dataset" + qualified_name = "aosp_dataset_fix_commits" + + @classmethod + def steps(cls): + return ( + cls.clone, + cls.collect_and_store_advisories, + cls.clean_downloads, + ) + + def clone(self): + self.repo_url = "git+https://github.com/quarkslab/aosp_dataset" + self.log(f"Cloning `{self.repo_url}`") + self.vcs_response = fetch_via_vcs(self.repo_url) + + def advisories_count(self): + root = Path(self.vcs_response.dest_dir) + return sum(1 for _ in root.rglob("*.json")) + + def collect_advisories(self): + self.log(f"Processing aosp_dataset fix commits.") + base_path = Path(self.vcs_response.dest_dir) / "cves" + for file_path in base_path.rglob("*.json"): + if not file_path.name.startswith("CVE-"): + continue + + with open(file_path) as f: + vulnerability_data = json.load(f) + + vulnerability_id = vulnerability_data.get("cveId", []) + if ( + not vulnerability_id or "," in vulnerability_id + ): # escape invalid multiple CVE-2017-13077, CVE-2017-13078 + continue + + summary = vulnerability_data.get("vulnerabilityType") + date_reported = vulnerability_data.get("dateReported") + date_published = dateparser.parse(date_reported) if date_reported else None + if date_published and not date_published.tzinfo: + date_published = date_published.replace(tzinfo=timezone.utc) + + severities = [] + severity_value = vulnerability_data.get("severity") + if severity_value: + severities.append( + VulnerabilitySeverity( + system=GENERIC, + value=severity_value, + ) + ) + + references = [] + for commit_data in vulnerability_data.get("fixes", []): + vcs_url = commit_data.get("patchUrl") + + if not vcs_url: + continue + + ref = ReferenceV2( + reference_type="commit", + url=vcs_url, + ) + references.append(ref) + + yield AdvisoryData( + advisory_id=vulnerability_id, + summary=summary, + references_v2=references, + severities=severities, + date_published=date_published, + url=f"https://raw.githubusercontent.com/quarkslab/aosp_dataset/refs/heads/master/cves/{file_path.name}", + ) + + def clean_downloads(self): + """Cleanup any temporary repository data.""" + self.log("Cleaning up local repository resources.") + if hasattr(self, "repo") and self.repo.working_dir: + shutil.rmtree(path=self.repo.working_dir) + + def on_failure(self): + """Ensure cleanup is always performed on failure.""" + self.clean_downloads() diff --git a/vulnerabilities/tests/pipelines/v2_importers/test_commits_aosp_dataset.py b/vulnerabilities/tests/pipelines/v2_importers/test_commits_aosp_dataset.py new file mode 100644 index 000000000..b641d58ab --- /dev/null +++ b/vulnerabilities/tests/pipelines/v2_importers/test_commits_aosp_dataset.py @@ -0,0 +1,28 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import os +from pathlib import Path +from unittest.mock import Mock + +import pytest + +from vulnerabilities.pipelines.v2_importers.aosp_importer import AospImporterPipeline +from vulnerabilities.tests import util_tests + +TEST_DATA = Path(__file__).parent.parent.parent / "test_data" / "aosp" + + +@pytest.mark.django_db +def test_aosp_advisories(): + expected_file = os.path.join(TEST_DATA, "aosp_advisoryv2-expected.json") + pipeline = AospImporterPipeline() + pipeline.vcs_response = Mock(dest_dir=TEST_DATA) + result = [adv.to_dict() for adv in pipeline.collect_advisories()] + util_tests.check_results_against_json(result, expected_file) diff --git a/vulnerabilities/tests/test_data/aosp/aosp_advisoryv2-expected.json b/vulnerabilities/tests/test_data/aosp/aosp_advisoryv2-expected.json new file mode 100644 index 000000000..956abb3dc --- /dev/null +++ b/vulnerabilities/tests/test_data/aosp/aosp_advisoryv2-expected.json @@ -0,0 +1,48 @@ +[ + { + "advisory_id": "CVE-2021-30294", + "aliases": [], + "summary": "Vulnerability", + "affected_packages": [], + "references_v2": [ + { + "reference_id": "", + "reference_type": "commit", + "url": "https://source.codeaurora.org/quic/la/kernel/msm-5.4/commit/?id=d6876813add62f3cac7c429a41cc8710005d69e8" + } + ], + "severities": [ + { + "system": "generic_textual", + "value": "High", + "scoring_elements": "" + } + ], + "date_published": null, + "weaknesses": [], + "url": "https://raw.githubusercontent.com/quarkslab/aosp_dataset/refs/heads/master/cves/CVE-aosp_test1.json" + }, + { + "advisory_id": "CVE-2017-13282", + "aliases": [], + "summary": "Remote Code Execution Vulnerability", + "affected_packages": [], + "references_v2": [ + { + "reference_id": "", + "reference_type": "commit", + "url": "https://android.googlesource.com/platform/system/bt/+/6ecbbc093f4383e90cbbf681cd55da1303a8ef94" + } + ], + "severities": [ + { + "system": "generic_textual", + "value": "Critical", + "scoring_elements": "" + } + ], + "date_published": "2018-04-04T00:00:00+00:00", + "weaknesses": [], + "url": "https://raw.githubusercontent.com/quarkslab/aosp_dataset/refs/heads/master/cves/CVE-aosp_test2.json" + } +] \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/aosp/cves/CVE-aosp_test1.json b/vulnerabilities/tests/test_data/aosp/cves/CVE-aosp_test1.json new file mode 100644 index 000000000..1fff64059 --- /dev/null +++ b/vulnerabilities/tests/test_data/aosp/cves/CVE-aosp_test1.json @@ -0,0 +1,14 @@ +{ + "cveId": "CVE-2021-30294", + "dateReported": null, + "vulnerabilityType": "Vulnerability", + "language": "c", + "fixes": [ + { + "commitId": "", + "patchUrl": "https://source.codeaurora.org/quic/la/kernel/msm-5.4/commit/?id=d6876813add62f3cac7c429a41cc8710005d69e8" + } + ], + "severity": "High", + "component": "Qualcomm Display" +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/aosp/cves/CVE-aosp_test2.json b/vulnerabilities/tests/test_data/aosp/cves/CVE-aosp_test2.json new file mode 100644 index 000000000..95201e382 --- /dev/null +++ b/vulnerabilities/tests/test_data/aosp/cves/CVE-aosp_test2.json @@ -0,0 +1,14 @@ +{ + "cveId": "CVE-2017-13282", + "dateReported": "2018-04-04", + "vulnerabilityType": "Remote Code Execution Vulnerability", + "language": "c", + "fixes": [ + { + "commitId": "6ecbbc093f4383e90cbbf681cd55da1303a8ef94", + "patchUrl": "https://android.googlesource.com/platform/system/bt/+/6ecbbc093f4383e90cbbf681cd55da1303a8ef94" + } + ], + "severity": "Critical", + "component": "System" +} \ No newline at end of file