Merge pull request #1066 from TG1999/migrate/project_kb_msr_2019

Migrate projectkbmsr2019 importer

Author: TG1999

CHANGELOG.rst

@@ -8,6 +8,7 @@ Next release
 - We re-enabled support for the mozilla vulnerabilities advisories importer.
 - We re-enabled support for the gentoo vulnerabilities advisories importer.
 - We re-enabled support for the istio vulnerabilities advisories importer.
+- We re-enabled support for the kbmsr2019 vulnerabilities advisories importer.
 
 
 Version v31.1.1

vulnerabilities/importers/__init__.py

@@ -22,6 +22,7 @@
 from vulnerabilities.importers import nvd
 from vulnerabilities.importers import openssl
 from vulnerabilities.importers import postgresql
+from vulnerabilities.importers import project_kb_msr2019
 from vulnerabilities.importers import pypa
 from vulnerabilities.importers import pysec
 from vulnerabilities.importers import redhat
@@ -49,6 +50,7 @@
     mozilla.MozillaImporter,
     gentoo.GentooImporter,
     istio.IstioImporter,
+    project_kb_msr2019.ProjectKBMSRImporter,
 ]
 
 IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY}

vulnerabilities/importers/project_kb_msr2019.py

@@ -7,13 +7,9 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
-import csv
-import urllib.request
-
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import Importer
 from vulnerabilities.importer import Reference
-from vulnerabilities.utils import create_etag
 from vulnerabilities.utils import is_cve
 
 # Reading CSV file from  a url using `requests` is bit too complicated.
@@ -23,42 +19,25 @@
 class ProjectKBMSRImporter(Importer):
 
     url = "https://raw.githubusercontent.com/SAP/project-kb/master/MSR2019/dataset/vulas_db_msr2019_release.csv"
+    spdx_license_expression = "Apache-2.0"
+    license_url = "https://github.com/SAP/project-kb/blob/main/LICENSE.txt"
 
-    def updated_advisories(self):
-        if create_etag(data_src=self, url=self.url, etag_key="ETag"):
-            raw_data = self.fetch()
-            advisories = self.to_advisories(raw_data)
-            return self.batch_advisories(advisories)
-
-        return []
-
-    def fetch(self):
-        response = urllib.request.urlopen(self.url)
-        lines = [l.decode("utf-8") for l in response.readlines()]
-        return csv.reader(lines)
+    def advisory_data(self):
+        raw_data = fetch_and_read_from_csv(self.url)
+        yield from self.to_advisories(raw_data)
 
-    @staticmethod
-    def to_advisories(csv_reader):
+    def to_advisories(self, csv_reader):
         # Project KB MSR csv file has no header row
-        advisories = []
         for row in csv_reader:
             vuln_id, proj_home, fix_commit, _ = row
             commit_link = proj_home + "/commit/" + fix_commit
 
-            if is_cve(vuln_id):
-                reference = Reference(url=commit_link)
+            if not is_cve(vuln_id):
+                continue
 
-            else:
-                reference = Reference(url=commit_link, reference_id=vuln_id)
-                vuln_id = ""
-
-            advisories.append(
-                AdvisoryData(
-                    summary="",
-                    affected_packages=[],
-                    references=[reference],
-                    vulnerability_id=vuln_id,
-                )
+            reference = Reference(url=commit_link)
+            yield AdvisoryData(
+                aliases=[vuln_id],
+                summary="",
+                references=[reference],
             )
-
-        return advisories

vulnerabilities/tests/conftest.py

@@ -30,7 +30,6 @@ def no_rmtree(monkeypatch):
     "test_api.py",
     "test_elixir_security.py",
     "test_models.py",
-    "test_msr2019.py",
     "test_package_managers.py",
     "test_ruby.py",
     "test_rust.py",