fix: CVEs from multiple sources missing (Fixes #2418) (#2421)

* Fixes #2418

Author: anthonyharrison

README.md

@@ -304,12 +304,6 @@ This data source contains CVEs pertaining to RedHat Products.
 
 Access to the data is subject to [Legal Notice](https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0/html/red_hat_security_data_api/legal-notice).
 
-### [RedHat Security Database](https://access.redhat.com/security/data) (REDHAT)
-
-This data source contains CVEs pertaining to RedHat Products.
-
-Access to the data is subject to [Legal Notice](https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0/html/red_hat_security_data_api/legal-notice).
-
 ## Binary checker list
 
 The following checkers are available for finding components in binary files:

cve_bin_tool/cve_scanner.py

@@ -222,6 +222,20 @@ def get_cves(self, product_info: ProductInfo, triage_data: TriageData):
                     # Skipping CVEs from disabled data sources
                     if row["data_source"] in self.disabled_sources:
                         continue
+
+                    # To avoid duplicate reporting, skip reporting CVE if already reported
+                    duplicate_found = False
+                    for c in cves:
+                        if c.cve_number == row["cve_number"]:
+                            self.logger.debug(
+                                f"{row['cve_number']} already reported from {c.data_source}"
+                            )
+                            duplicate_found = True
+                            break
+
+                    if duplicate_found:
+                        continue
+
                     triage = triage_data.get(row["cve_number"]) or triage_data.get(
                         "default"
                     )

cve_bin_tool/cvedb.py

@@ -226,7 +226,7 @@ def table_schemas(self):
             cvss_vector TEXT,
             data_source TEXT,
             last_modified TIMESTAMP,
-            PRIMARY KEY(cve_number)
+            PRIMARY KEY(cve_number, data_source)
         )
         """
         version_range_create = """
@@ -239,6 +239,7 @@ def table_schemas(self):
             versionStartExcluding TEXT,
             versionEndIncluding TEXT,
             versionEndExcluding TEXT,
+            data_source TEXT,
             FOREIGN KEY(cve_number) REFERENCES cve_severity(cve_number)
         )
         """
@@ -295,6 +296,7 @@ def populate_db(self) -> None:
                 self.populate_affected(
                     affected_data,
                     cursor,
+                    data_source=source_name,
                 )
             if self.connection is not None:
                 self.connection.commit()
@@ -357,7 +359,7 @@ def populate_severity(self, severity_data, cursor, data_source):
         # Delete any old range entries for this CVE_number
         cursor.executemany(del_cve_range, [(cve["ID"],) for cve in severity_data])
 
-    def populate_affected(self, affected_data, cursor):
+    def populate_affected(self, affected_data, cursor, data_source):
 
         insert_cve_range = """
         INSERT or REPLACE INTO cve_range(
@@ -368,26 +370,31 @@ def populate_affected(self, affected_data, cursor):
             versionStartIncluding,
             versionStartExcluding,
             versionEndIncluding,
-            versionEndExcluding
-        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+            versionEndExcluding,
+            data_source
+        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
         """
 
-        cursor.executemany(
-            insert_cve_range,
-            [
-                (
-                    affected["cve_id"],
-                    affected["vendor"],
-                    affected["product"],
-                    affected["version"],
-                    affected["versionStartIncluding"],
-                    affected["versionStartExcluding"],
-                    affected["versionEndIncluding"],
-                    affected["versionEndExcluding"],
-                )
-                for affected in affected_data
-            ],
-        )
+        try:
+            cursor.executemany(
+                insert_cve_range,
+                [
+                    (
+                        affected["cve_id"],
+                        affected["vendor"],
+                        affected["product"],
+                        affected["version"],
+                        affected["versionStartIncluding"],
+                        affected["versionStartExcluding"],
+                        affected["versionEndIncluding"],
+                        affected["versionEndExcluding"],
+                        data_source,
+                    )
+                    for affected in affected_data
+                ],
+            )
+        except Exception as e:
+            LOGGER.info(f"Unable to insert data for {data_source} - {e}")
 
     def clear_cached_data(self) -> None:
         self.create_cache_backup()

cve_bin_tool/data_sources/gad_source.py

@@ -88,6 +88,8 @@ async def fetch_cves(self):
 
         if not Path(self.gad_path).exists():
             Path(self.gad_path).mkdir()
+            # As no data, force full update
+            self.incremental_update = False
 
         if not self.session:
             connector = aiohttp.TCPConnector(limit_per_host=19)
@@ -184,7 +186,7 @@ def parse_range_string(self, range_string):
                 "versionEndExcluding": "",
             }
 
-            versions = version_string.split(",")
+            versions = version_string.replace(",", " ").split(" ")
 
             for version in versions:
                 if ">=" in version:

cve_bin_tool/data_sources/osv_source.py

@@ -161,6 +161,8 @@ async def fetch_cves(self):
 
         if not Path(self.osv_path).exists():
             Path(self.osv_path).mkdir()
+            # As no data, force full update
+            self.incremental_update = False
 
         if not self.session:
             connector = aiohttp.TCPConnector(limit_per_host=19)

cve_bin_tool/data_sources/redhat_source.py

@@ -149,7 +149,7 @@ def format_data(self, all_cve_entries):
             4: "MEDIUM",
             5: "MEDIUM",
             6: "MEDIUM",
-            7: "MEDIUM",
+            7: "HIGH",
             8: "HIGH",
             9: "CRITICAL",
             10: "CRITICAL",
@@ -222,25 +222,23 @@ def format_data(self, all_cve_entries):
                         s = package.split(":")
                         product = s[0]
                         version = s[1]
+                        affected = {
+                            "cve_id": cve_id,
+                            "vendor": "redhat",
+                            "product": product,
+                            "version": version,
+                            "versionStartIncluding": "",
+                            "versionStartExcluding": "",
+                            "versionEndIncluding": "",
+                            "versionEndExcluding": "",
+                        }
+                        affected_data.append(affected)
+                        if cve_to_write:
+                            severity_data.append(cve)
+                            cve_to_write = False
                     else:
                         # Version not specified
                         LOGGER.debug(f"{cve_id} : Version not specified for {package}")
-                        product = package
-                        version = "*"
-                    affected = {
-                        "cve_id": cve_id,
-                        "vendor": "redhat",
-                        "product": product,
-                        "version": version,
-                        "versionStartIncluding": "",
-                        "versionStartExcluding": "",
-                        "versionEndIncluding": "",
-                        "versionEndExcluding": "",
-                    }
-                    affected_data.append(affected)
-                    if cve_to_write:
-                        severity_data.append(cve)
-                        cve_to_write = False
                 except Exception as e:
                     LOGGER.debug(e)
                     LOGGER.debug(f"{cve_id} : Affected {package}")

test/test_requirements.py

@@ -150,7 +150,6 @@ def test_requirements():
         output_json = json.load(f)
         for entry in output_json:
             assert entry["remarks"] in ["Mitigated", "Ignored"]
-
     # Disabled until we fix how ignored/mitigated issues are listed
     # See https://github.com/intel/cve-bin-tool/issues/1752
     # assert (