From bf38a7b6b0b32c19c5ad74e4c0b9a86436352ba9 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Tue, 26 Aug 2025 00:06:53 +0300 Subject: [PATCH 1/4] Add initial support for collecting aosp commits. Signed-off-by: ziad hany --- vulnerabilities/improvers/__init__.py | 4 + .../collect_commits_aosp_dataset.py | 84 +++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 vulnerabilities/pipelines/v2_improvers/collect_commits_aosp_dataset.py diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index 1be791241..768b0e93e 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -19,6 +19,9 @@ from vulnerabilities.pipelines import flag_ghost_packages from vulnerabilities.pipelines import populate_vulnerability_summary_pipeline from vulnerabilities.pipelines import remove_duplicate_advisories +from vulnerabilities.pipelines.v2_improvers import ( + collect_commits_aosp_dataset as collect_commits_aosp_v2, +) from vulnerabilities.pipelines.v2_improvers import compute_advisory_todo as compute_advisory_todo_v2 from vulnerabilities.pipelines.v2_improvers import compute_package_risk as compute_package_risk_v2 from vulnerabilities.pipelines.v2_improvers import ( @@ -68,5 +71,6 @@ compute_version_rank_v2.ComputeVersionRankPipeline, compute_advisory_todo_v2.ComputeToDo, compute_advisory_todo.ComputeToDo, + collect_commits_aosp_v2.CollectFixCommitsAospDatasetPipeline, ] ) diff --git a/vulnerabilities/pipelines/v2_improvers/collect_commits_aosp_dataset.py b/vulnerabilities/pipelines/v2_improvers/collect_commits_aosp_dataset.py new file mode 100644 index 000000000..2b89bc7db --- /dev/null +++ b/vulnerabilities/pipelines/v2_improvers/collect_commits_aosp_dataset.py @@ -0,0 +1,84 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import json +from pathlib import Path + +from fetchcode.vcs import fetch_via_vcs + +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import CodeFixV2 +from vulnerabilities.pipelines import VulnerableCodePipeline + + +class CollectFixCommitsAospDatasetPipeline(VulnerableCodePipeline): + """ + Pipeline to collect fix commits from Aosp Dataset: + """ + + pipeline_id = "aosp_dataset_fix_commits" + spdx_license_expression = "Apache-2.0" + license_url = "https://github.com/quarkslab/aosp_dataset/blob/master/LICENSE" + importer_name = "aosp_dataset" + qualified_name = "aosp_dataset_fix_commits" + repo_url = "git+https://github.com/quarkslab/aosp_dataset" + + @classmethod + def steps(cls): + return ( + cls.clone, + cls.collect_fix_commits, + ) + + def clone(self): + self.log(f"Cloning `{self.repo_url}`") + self.vcs_response = fetch_via_vcs(self.repo_url) + + def collect_fix_commits(self): + self.log(f"Processing aosp_dataset fix commits.") + base_path = Path(self.vcs_response.dest_dir) / "cves" + for file_path in base_path.rglob("*.json"): + if not file_path.name.startswith("CVE-"): + continue + + with open(file_path) as f: + vulnerability_data = json.load(f) + + vulnerability_id = vulnerability_data.get("cveId") + if not vulnerability_id: + continue + + try: + advisories = AdvisoryV2.objects.filter(advisory_id__iendswith=vulnerability_id) + except AdvisoryV2.DoesNotExist: + self.log(f"Can't find vulnerability_id: {vulnerability_id}") + continue + + for advisory in advisories: + for commit_data in vulnerability_data.get("fixes", []): + vcs_url = commit_data.get("patchUrl") + for impact in advisory.impacted_packages.all(): + for package in impact.affecting_packages.all(): + code_fix, created = CodeFixV2.objects.get_or_create( + commits=[vcs_url], + advisory=advisory, + affected_package=package, + ) + + if created: + self.log( + f"Created CodeFix entry for vulnerability_id: {vulnerability_id} with VCS URL {vcs_url}" + ) + + def clean_downloads(self): + if self.vcs_response: + self.log(f"Removing cloned repository") + self.vcs_response.delete() + + def on_failure(self): + self.clean_downloads() From f0640337ccdd29c55b303ea769bafe7c05e890f1 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Tue, 21 Oct 2025 18:25:41 +0300 Subject: [PATCH 2/4] Add a test for the aosp importer Signed-off-by: ziad hany --- vulnerabilities/importers/__init__.py | 2 + vulnerabilities/improvers/__init__.py | 4 - .../pipelines/v2_importers/aosp_importer.py | 97 +++++++++++++++++++ .../collect_commits_aosp_dataset.py | 84 ---------------- .../v2_importers/test_commits_aosp_dataset.py | 28 ++++++ .../aosp/aosp_advisoryv2-expected.json | 36 +++++++ .../test_data/aosp/cves/CVE-aosp_test1.json | 14 +++ .../test_data/aosp/cves/CVE-aosp_test2.json | 14 +++ 8 files changed, 191 insertions(+), 88 deletions(-) create mode 100644 vulnerabilities/pipelines/v2_importers/aosp_importer.py delete mode 100644 vulnerabilities/pipelines/v2_improvers/collect_commits_aosp_dataset.py create mode 100644 vulnerabilities/tests/pipelines/v2_importers/test_commits_aosp_dataset.py create mode 100644 vulnerabilities/tests/test_data/aosp/aosp_advisoryv2-expected.json create mode 100644 vulnerabilities/tests/test_data/aosp/cves/CVE-aosp_test1.json create mode 100644 vulnerabilities/tests/test_data/aosp/cves/CVE-aosp_test2.json diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index 82ee4525a..73ef6ef01 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -41,6 +41,7 @@ from vulnerabilities.pipelines import nvd_importer from vulnerabilities.pipelines import pypa_importer from vulnerabilities.pipelines import pysec_importer +from vulnerabilities.pipelines.v2_importers import aosp_importer from vulnerabilities.pipelines.v2_importers import apache_httpd_importer as apache_httpd_v2 from vulnerabilities.pipelines.v2_importers import archlinux_importer as archlinux_importer_v2 from vulnerabilities.pipelines.v2_importers import curl_importer as curl_importer_v2 @@ -81,6 +82,7 @@ mozilla_importer_v2.MozillaImporterPipeline, github_osv_importer_v2.GithubOSVImporterPipeline, redhat_importer_v2.RedHatImporterPipeline, + aosp_importer.AospImporterPipeline, nvd_importer.NVDImporterPipeline, github_importer.GitHubAPIImporterPipeline, gitlab_importer.GitLabImporterPipeline, diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index 768b0e93e..1be791241 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -19,9 +19,6 @@ from vulnerabilities.pipelines import flag_ghost_packages from vulnerabilities.pipelines import populate_vulnerability_summary_pipeline from vulnerabilities.pipelines import remove_duplicate_advisories -from vulnerabilities.pipelines.v2_improvers import ( - collect_commits_aosp_dataset as collect_commits_aosp_v2, -) from vulnerabilities.pipelines.v2_improvers import compute_advisory_todo as compute_advisory_todo_v2 from vulnerabilities.pipelines.v2_improvers import compute_package_risk as compute_package_risk_v2 from vulnerabilities.pipelines.v2_improvers import ( @@ -71,6 +68,5 @@ compute_version_rank_v2.ComputeVersionRankPipeline, compute_advisory_todo_v2.ComputeToDo, compute_advisory_todo.ComputeToDo, - collect_commits_aosp_v2.CollectFixCommitsAospDatasetPipeline, ] ) diff --git a/vulnerabilities/pipelines/v2_importers/aosp_importer.py b/vulnerabilities/pipelines/v2_importers/aosp_importer.py new file mode 100644 index 000000000..7262edb44 --- /dev/null +++ b/vulnerabilities/pipelines/v2_importers/aosp_importer.py @@ -0,0 +1,97 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import json +import shutil +from pathlib import Path + +import dateparser +from django.core.exceptions import ValidationError +from fetchcode.vcs import fetch_via_vcs + +from vulnerabilities.importer import AdvisoryData +from vulnerabilities.importer import ReferenceV2 +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 + + +class AospImporterPipeline(VulnerableCodeBaseImporterPipelineV2): + """ + Pipeline to collect fix commits from Aosp Dataset: + """ + + pipeline_id = "aosp_dataset_fix_commits" + spdx_license_expression = "Apache-2.0" + license_url = "https://github.com/quarkslab/aosp_dataset/blob/master/LICENSE" + importer_name = "aosp_dataset" + qualified_name = "aosp_dataset_fix_commits" + + @classmethod + def steps(cls): + return ( + cls.clone, + cls.collect_and_store_advisories, + cls.clean_downloads, + ) + + def clone(self): + self.repo_url = "git+https://github.com/quarkslab/aosp_dataset" + self.log(f"Cloning `{self.repo_url}`") + self.vcs_response = fetch_via_vcs(self.repo_url) + + def advisories_count(self): + root = Path(self.vcs_response.dest_dir) + return sum(1 for _ in root.rglob("*.json")) + + def collect_advisories(self): + self.log(f"Processing aosp_dataset fix commits.") + base_path = Path(self.vcs_response.dest_dir) / "cves" + for file_path in base_path.rglob("*.json"): + if not file_path.name.startswith("CVE-"): + continue + + with open(file_path) as f: + vulnerability_data = json.load(f) + + vulnerability_id = vulnerability_data.get("cveId", []) + if ( + not vulnerability_id or "," in vulnerability_id + ): # escape invalid multiple CVE-2017-13077, CVE-2017-13078 + continue + + summary = vulnerability_data.get("vulnerabilityType") + date_reported = vulnerability_data.get("dateReported") + date_published = dateparser.parse(date_reported) if date_reported else None + + references = [] + for commit_data in vulnerability_data.get("fixes", []): + vcs_url = commit_data.get("patchUrl") + + if not vcs_url: + continue + + ref = ReferenceV2(reference_type="commit", url=vcs_url) + references.append(ref) + + yield AdvisoryData( + advisory_id=vulnerability_id, + summary=summary, + references_v2=references, + date_published=date_published, + url=f"https://raw.githubusercontent.com/quarkslab/aosp_dataset/refs/heads/master/cves/{file_path.name}", + ) + + def clean_downloads(self): + """Cleanup any temporary repository data.""" + self.log("Cleaning up local repository resources.") + if hasattr(self, "repo") and self.repo.working_dir: + shutil.rmtree(path=self.repo.working_dir) + + def on_failure(self): + """Ensure cleanup is always performed on failure.""" + self.clean_downloads() diff --git a/vulnerabilities/pipelines/v2_improvers/collect_commits_aosp_dataset.py b/vulnerabilities/pipelines/v2_improvers/collect_commits_aosp_dataset.py deleted file mode 100644 index 2b89bc7db..000000000 --- a/vulnerabilities/pipelines/v2_improvers/collect_commits_aosp_dataset.py +++ /dev/null @@ -1,84 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import json -from pathlib import Path - -from fetchcode.vcs import fetch_via_vcs - -from vulnerabilities.models import AdvisoryV2 -from vulnerabilities.models import CodeFixV2 -from vulnerabilities.pipelines import VulnerableCodePipeline - - -class CollectFixCommitsAospDatasetPipeline(VulnerableCodePipeline): - """ - Pipeline to collect fix commits from Aosp Dataset: - """ - - pipeline_id = "aosp_dataset_fix_commits" - spdx_license_expression = "Apache-2.0" - license_url = "https://github.com/quarkslab/aosp_dataset/blob/master/LICENSE" - importer_name = "aosp_dataset" - qualified_name = "aosp_dataset_fix_commits" - repo_url = "git+https://github.com/quarkslab/aosp_dataset" - - @classmethod - def steps(cls): - return ( - cls.clone, - cls.collect_fix_commits, - ) - - def clone(self): - self.log(f"Cloning `{self.repo_url}`") - self.vcs_response = fetch_via_vcs(self.repo_url) - - def collect_fix_commits(self): - self.log(f"Processing aosp_dataset fix commits.") - base_path = Path(self.vcs_response.dest_dir) / "cves" - for file_path in base_path.rglob("*.json"): - if not file_path.name.startswith("CVE-"): - continue - - with open(file_path) as f: - vulnerability_data = json.load(f) - - vulnerability_id = vulnerability_data.get("cveId") - if not vulnerability_id: - continue - - try: - advisories = AdvisoryV2.objects.filter(advisory_id__iendswith=vulnerability_id) - except AdvisoryV2.DoesNotExist: - self.log(f"Can't find vulnerability_id: {vulnerability_id}") - continue - - for advisory in advisories: - for commit_data in vulnerability_data.get("fixes", []): - vcs_url = commit_data.get("patchUrl") - for impact in advisory.impacted_packages.all(): - for package in impact.affecting_packages.all(): - code_fix, created = CodeFixV2.objects.get_or_create( - commits=[vcs_url], - advisory=advisory, - affected_package=package, - ) - - if created: - self.log( - f"Created CodeFix entry for vulnerability_id: {vulnerability_id} with VCS URL {vcs_url}" - ) - - def clean_downloads(self): - if self.vcs_response: - self.log(f"Removing cloned repository") - self.vcs_response.delete() - - def on_failure(self): - self.clean_downloads() diff --git a/vulnerabilities/tests/pipelines/v2_importers/test_commits_aosp_dataset.py b/vulnerabilities/tests/pipelines/v2_importers/test_commits_aosp_dataset.py new file mode 100644 index 000000000..ffdb5050b --- /dev/null +++ b/vulnerabilities/tests/pipelines/v2_importers/test_commits_aosp_dataset.py @@ -0,0 +1,28 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import os +from pathlib import Path +from unittest.mock import Mock + +import pytest + +from vulnerabilities.pipelines.v2_importers.aosp_importer import AospImporterPipeline +from vulnerabilities.tests import util_tests + +TEST_DATA = Path(__file__).parent.parent.parent / "test_data" / "aosp" + + +@pytest.mark.django_db +def test_aosp_advisories1(): + expected_file = os.path.join(TEST_DATA, "aosp_advisoryv2-expected.json") + pipeline = AospImporterPipeline() + pipeline.vcs_response = Mock(dest_dir=TEST_DATA) + result = [adv.to_dict() for adv in pipeline.collect_advisories()] + util_tests.check_results_against_json(result, expected_file) diff --git a/vulnerabilities/tests/test_data/aosp/aosp_advisoryv2-expected.json b/vulnerabilities/tests/test_data/aosp/aosp_advisoryv2-expected.json new file mode 100644 index 000000000..fed4befe6 --- /dev/null +++ b/vulnerabilities/tests/test_data/aosp/aosp_advisoryv2-expected.json @@ -0,0 +1,36 @@ +[ + { + "advisory_id": "CVE-2021-30294", + "aliases": [], + "summary": "Vulnerability", + "affected_packages": [], + "references_v2": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://source.codeaurora.org/quic/la/kernel/msm-5.4/commit/?id=d6876813add62f3cac7c429a41cc8710005d69e8" + } + ], + "severities": [], + "date_published": null, + "weaknesses": [], + "url": "https://raw.githubusercontent.com/quarkslab/aosp_dataset/refs/heads/master/cves/CVE-aosp_test1.json" + }, + { + "advisory_id": "CVE-2017-13282", + "aliases": [], + "summary": "Remote Code Execution Vulnerability", + "affected_packages": [], + "references_v2": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://android.googlesource.com/platform/system/bt/+/6ecbbc093f4383e90cbbf681cd55da1303a8ef94" + } + ], + "severities": [], + "date_published": "2018-04-04T00:00:00", + "weaknesses": [], + "url": "https://raw.githubusercontent.com/quarkslab/aosp_dataset/refs/heads/master/cves/CVE-aosp_test2.json" + } +] \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/aosp/cves/CVE-aosp_test1.json b/vulnerabilities/tests/test_data/aosp/cves/CVE-aosp_test1.json new file mode 100644 index 000000000..1fff64059 --- /dev/null +++ b/vulnerabilities/tests/test_data/aosp/cves/CVE-aosp_test1.json @@ -0,0 +1,14 @@ +{ + "cveId": "CVE-2021-30294", + "dateReported": null, + "vulnerabilityType": "Vulnerability", + "language": "c", + "fixes": [ + { + "commitId": "", + "patchUrl": "https://source.codeaurora.org/quic/la/kernel/msm-5.4/commit/?id=d6876813add62f3cac7c429a41cc8710005d69e8" + } + ], + "severity": "High", + "component": "Qualcomm Display" +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/aosp/cves/CVE-aosp_test2.json b/vulnerabilities/tests/test_data/aosp/cves/CVE-aosp_test2.json new file mode 100644 index 000000000..95201e382 --- /dev/null +++ b/vulnerabilities/tests/test_data/aosp/cves/CVE-aosp_test2.json @@ -0,0 +1,14 @@ +{ + "cveId": "CVE-2017-13282", + "dateReported": "2018-04-04", + "vulnerabilityType": "Remote Code Execution Vulnerability", + "language": "c", + "fixes": [ + { + "commitId": "6ecbbc093f4383e90cbbf681cd55da1303a8ef94", + "patchUrl": "https://android.googlesource.com/platform/system/bt/+/6ecbbc093f4383e90cbbf681cd55da1303a8ef94" + } + ], + "severity": "Critical", + "component": "System" +} \ No newline at end of file From fc42808e56816a126529cc2b8d98af0a2ce19813 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Fri, 24 Oct 2025 18:23:38 +0300 Subject: [PATCH 3/4] Add missing severity Update reference_type in the expected advisory file Signed-off-by: ziad hany --- .../pipelines/v2_importers/aosp_importer.py | 18 ++++++++++++++++- .../aosp/aosp_advisoryv2-expected.json | 20 +++++++++++++++---- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/vulnerabilities/pipelines/v2_importers/aosp_importer.py b/vulnerabilities/pipelines/v2_importers/aosp_importer.py index 7262edb44..67e6ec3f0 100644 --- a/vulnerabilities/pipelines/v2_importers/aosp_importer.py +++ b/vulnerabilities/pipelines/v2_importers/aosp_importer.py @@ -17,7 +17,9 @@ from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import ReferenceV2 +from vulnerabilities.importer import VulnerabilitySeverity from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 +from vulnerabilities.severity_systems import GENERIC class AospImporterPipeline(VulnerableCodeBaseImporterPipelineV2): @@ -68,6 +70,16 @@ def collect_advisories(self): date_reported = vulnerability_data.get("dateReported") date_published = dateparser.parse(date_reported) if date_reported else None + severities = [] + severity_value = vulnerability_data.get("severity") + if severity_value: + severities.append( + VulnerabilitySeverity( + system=GENERIC, + value=severity_value, + ) + ) + references = [] for commit_data in vulnerability_data.get("fixes", []): vcs_url = commit_data.get("patchUrl") @@ -75,13 +87,17 @@ def collect_advisories(self): if not vcs_url: continue - ref = ReferenceV2(reference_type="commit", url=vcs_url) + ref = ReferenceV2( + reference_type="commit", + url=vcs_url, + ) references.append(ref) yield AdvisoryData( advisory_id=vulnerability_id, summary=summary, references_v2=references, + severities=severities, date_published=date_published, url=f"https://raw.githubusercontent.com/quarkslab/aosp_dataset/refs/heads/master/cves/{file_path.name}", ) diff --git a/vulnerabilities/tests/test_data/aosp/aosp_advisoryv2-expected.json b/vulnerabilities/tests/test_data/aosp/aosp_advisoryv2-expected.json index fed4befe6..28d9f446a 100644 --- a/vulnerabilities/tests/test_data/aosp/aosp_advisoryv2-expected.json +++ b/vulnerabilities/tests/test_data/aosp/aosp_advisoryv2-expected.json @@ -7,11 +7,17 @@ "references_v2": [ { "reference_id": "", - "reference_type": "", + "reference_type": "commit", "url": "https://source.codeaurora.org/quic/la/kernel/msm-5.4/commit/?id=d6876813add62f3cac7c429a41cc8710005d69e8" } ], - "severities": [], + "severities": [ + { + "system": "generic_textual", + "value": "High", + "scoring_elements": "" + } + ], "date_published": null, "weaknesses": [], "url": "https://raw.githubusercontent.com/quarkslab/aosp_dataset/refs/heads/master/cves/CVE-aosp_test1.json" @@ -24,11 +30,17 @@ "references_v2": [ { "reference_id": "", - "reference_type": "", + "reference_type": "commit", "url": "https://android.googlesource.com/platform/system/bt/+/6ecbbc093f4383e90cbbf681cd55da1303a8ef94" } ], - "severities": [], + "severities": [ + { + "system": "generic_textual", + "value": "Critical", + "scoring_elements": "" + } + ], "date_published": "2018-04-04T00:00:00", "weaknesses": [], "url": "https://raw.githubusercontent.com/quarkslab/aosp_dataset/refs/heads/master/cves/CVE-aosp_test2.json" From 6e29d6f11035a652ea0b88d2da7734c780263666 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Fri, 24 Oct 2025 21:34:25 +0300 Subject: [PATCH 4/4] Update date_published time parser Signed-off-by: ziad hany --- vulnerabilities/pipelines/v2_importers/aosp_importer.py | 4 +++- .../tests/pipelines/v2_importers/test_commits_aosp_dataset.py | 2 +- .../tests/test_data/aosp/aosp_advisoryv2-expected.json | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/vulnerabilities/pipelines/v2_importers/aosp_importer.py b/vulnerabilities/pipelines/v2_importers/aosp_importer.py index 67e6ec3f0..391be0df7 100644 --- a/vulnerabilities/pipelines/v2_importers/aosp_importer.py +++ b/vulnerabilities/pipelines/v2_importers/aosp_importer.py @@ -9,10 +9,10 @@ import json import shutil +from datetime import timezone from pathlib import Path import dateparser -from django.core.exceptions import ValidationError from fetchcode.vcs import fetch_via_vcs from vulnerabilities.importer import AdvisoryData @@ -69,6 +69,8 @@ def collect_advisories(self): summary = vulnerability_data.get("vulnerabilityType") date_reported = vulnerability_data.get("dateReported") date_published = dateparser.parse(date_reported) if date_reported else None + if date_published and not date_published.tzinfo: + date_published = date_published.replace(tzinfo=timezone.utc) severities = [] severity_value = vulnerability_data.get("severity") diff --git a/vulnerabilities/tests/pipelines/v2_importers/test_commits_aosp_dataset.py b/vulnerabilities/tests/pipelines/v2_importers/test_commits_aosp_dataset.py index ffdb5050b..b641d58ab 100644 --- a/vulnerabilities/tests/pipelines/v2_importers/test_commits_aosp_dataset.py +++ b/vulnerabilities/tests/pipelines/v2_importers/test_commits_aosp_dataset.py @@ -20,7 +20,7 @@ @pytest.mark.django_db -def test_aosp_advisories1(): +def test_aosp_advisories(): expected_file = os.path.join(TEST_DATA, "aosp_advisoryv2-expected.json") pipeline = AospImporterPipeline() pipeline.vcs_response = Mock(dest_dir=TEST_DATA) diff --git a/vulnerabilities/tests/test_data/aosp/aosp_advisoryv2-expected.json b/vulnerabilities/tests/test_data/aosp/aosp_advisoryv2-expected.json index 28d9f446a..956abb3dc 100644 --- a/vulnerabilities/tests/test_data/aosp/aosp_advisoryv2-expected.json +++ b/vulnerabilities/tests/test_data/aosp/aosp_advisoryv2-expected.json @@ -41,7 +41,7 @@ "scoring_elements": "" } ], - "date_published": "2018-04-04T00:00:00", + "date_published": "2018-04-04T00:00:00+00:00", "weaknesses": [], "url": "https://raw.githubusercontent.com/quarkslab/aosp_dataset/refs/heads/master/cves/CVE-aosp_test2.json" }