Sort Debian OSV pkgInfos (#2120)

- Sorts the Debian OSV outputs to remain consistent each runs
(#2103)
- Fixes the `AffectedVersions`
- modifies `first_package_finder.py` to support future Debian releases,
like Debian 13.

Author: hogo6002

vulnfeeds/cmd/debian/main.go

@@ -6,6 +6,8 @@ import (
 	"net/http"
 	"os"
 	"path"
+	"sort"
+	"strconv"
 
 	"github.com/google/osv/vulnfeeds/cves"
 	"github.com/google/osv/vulnfeeds/utility"
@@ -81,25 +83,29 @@ func getDebianReleaseMap() (map[string]string, error) {
 	}
 
 	for _, row := range data[1:] {
-		if row[seriesIndex] == "experimental" || row[seriesIndex] == "sid" {
+		if row[versionIndex] == "" {
 			continue
 		}
-
 		releaseMap[row[seriesIndex]] = row[versionIndex]
 	}
 
 	return releaseMap, err
 }
 
 // updateOSVPkgInfos adds new release entries to osvPkgInfos.
-func updateOSVPkgInfos(pkgName string, cveId string, releases map[string]Release, osvPkgInfos map[string][]vulns.PackageInfo, debianReleaseMap map[string]string) {
+func updateOSVPkgInfos(pkgName string, cveId string, releases map[string]Release, osvPkgInfos map[string][]vulns.PackageInfo, debianReleaseMap map[string]string, releaseNames []string) {
 	var pkgInfos []vulns.PackageInfo
 	if value, ok := osvPkgInfos[cveId]; ok {
 		pkgInfos = value
 	}
-	for releaseName, release := range releases {
+
+	for _, releaseName := range releaseNames {
 		// Skips 'not yet assigned' entries because their status may change in the future.
 		// For reference on urgency levels, see: https://security-team.debian.org/security_tracker.html#severity-levels
+		release, ok := releases[releaseName]
+		if !ok {
+			continue
+		}
 		if release.Urgency == "not yet assigned" {
 			continue
 		}
@@ -114,17 +120,14 @@ func updateOSVPkgInfos(pkgName string, cveId string, releases map[string]Release
 		}
 		pkgInfo.EcosystemSpecific = make(map[string]string)
 
+		pkgInfo.VersionInfo = cves.VersionInfo{
+			AffectedVersions: []cves.AffectedVersion{{Introduced: "0"}},
+		}
 		if release.Status == "resolved" {
 			if release.FixedVersion == "0" { // not affected
 				continue
 			}
-			pkgInfo.VersionInfo = cves.VersionInfo{
-				AffectedVersions: []cves.AffectedVersion{{Introduced: "0", Fixed: release.FixedVersion}},
-			}
-		} else {
-			pkgInfo.VersionInfo = cves.VersionInfo{
-				AffectedVersions: []cves.AffectedVersion{{Introduced: "0"}},
-			}
+			pkgInfo.VersionInfo.AffectedVersions = append(pkgInfo.VersionInfo.AffectedVersions, cves.AffectedVersion{Fixed: release.FixedVersion})
 		}
 		pkgInfo.EcosystemSpecific["urgency"] = release.Urgency
 		pkgInfos = append(pkgInfos, pkgInfo)
@@ -138,9 +141,30 @@ func updateOSVPkgInfos(pkgName string, cveId string, releases map[string]Release
 func generateDebianSecurityTrackerOSV(debianData DebianSecurityTrackerData, debianReleaseMap map[string]string) map[string][]vulns.PackageInfo {
 	Logger.Infof("Converting Debian Security Tracker data to OSV package infos.")
 	osvPkgInfos := make(map[string][]vulns.PackageInfo)
-	for pkgName, pkg := range debianData {
+
+	// Sorts packages to ensure results remain consistent between runs.
+	var pkgNames []string
+	for name := range debianData {
+		pkgNames = append(pkgNames, name)
+	}
+	sort.Strings(pkgNames)
+
+	// Sorts releases to ensure pkgInfos remain consistent between runs.
+	releaseNames := make([]string, 0, len(debianReleaseMap))
+	for k := range debianReleaseMap {
+		releaseNames = append(releaseNames, k)
+	}
+
+	sort.Slice(releaseNames, func(i, j int) bool {
+		vi, _ := strconv.ParseFloat(debianReleaseMap[releaseNames[i]], 64)
+		vj, _ := strconv.ParseFloat(debianReleaseMap[releaseNames[j]], 64)
+		return vi < vj
+	})
+
+	for _, pkgName := range pkgNames {
+		pkg := debianData[pkgName]
 		for cveId, cve := range pkg {
-			updateOSVPkgInfos(pkgName, cveId, cve.Releases, osvPkgInfos, debianReleaseMap)
+			updateOSVPkgInfos(pkgName, cveId, cve.Releases, osvPkgInfos, debianReleaseMap, releaseNames)
 		}
 	}
 

vulnfeeds/cmd/debian/main_test.go

@@ -18,11 +18,12 @@ func Test_generateDebianSecurityTrackerOSV(t *testing.T) {
 	_ = json.NewDecoder(file).Decode(&decodedDebianData)
 
 	debianReleaseMap := make(map[string]string)
+	debianReleaseMap["trixie"] = "13"
 	debianReleaseMap["buster"] = "10"
 	debianReleaseMap["bullseye"] = "11"
 	debianReleaseMap["bookworm"] = "12"
-	debianReleaseMap["trixie"] = "13"
-	debianReleaseMap["forky"] = "14"
+	debianReleaseMap["sarge"] = "3.1"
+	debianReleaseMap["stretch"] = "9"
 
 	osvPkgInfos := generateDebianSecurityTrackerOSV(decodedDebianData, debianReleaseMap)
 	expectedCount := 2

vulnfeeds/test_data/parts/debian/CVE-2016-1585.debian.json

@@ -1,33 +1,54 @@
 [
   {
     "pkg_name": "apparmor",
-    "ecosystem": "Debian12",
-    "fixed_version": {},
+    "ecosystem": "Debian:10",
+    "fixed_version": {
+      "affected_versions": [
+        {
+          "introduced": "0"
+        }
+      ]
+    },
     "ecosystem_specific": {
       "urgency": "unimportant"
     }
   },
   {
     "pkg_name": "apparmor",
-    "ecosystem": "Debian11",
-    "fixed_version": {},
+    "ecosystem": "Debian:11",
+    "fixed_version": {
+      "affected_versions": [
+        {
+          "introduced": "0"
+        }
+      ]
+    },
     "ecosystem_specific": {
       "urgency": "unimportant"
     }
   },
   {
     "pkg_name": "apparmor",
-    "ecosystem": "Debian10",
-    "fixed_version": {},
+    "ecosystem": "Debian:12",
+    "fixed_version": {
+      "affected_versions": [
+        {
+          "introduced": "0"
+        }
+      ]
+    },
     "ecosystem_specific": {
       "urgency": "unimportant"
     }
   },
   {
     "pkg_name": "apparmor",
-    "ecosystem": "Debian13",
+    "ecosystem": "Debian:13",
     "fixed_version": {
       "affected_versions": [
+        {
+          "introduced": "0"
+        },
         {
           "fixed": "3.0.12-1"
         }
@@ -37,4 +58,4 @@
       "urgency": "unimportant"
     }
   }
-]
+]

vulnfeeds/test_data/parts/debian/CVE-2018-1000500.debian.json

@@ -1,32 +1,56 @@
 [
   {
     "pkg_name": "busybox",
-    "ecosystem": "Debian10",
-    "fixed_version": {},
+    "ecosystem": "Debian:10",
+    "fixed_version": {
+      "affected_versions": [
+        {
+          "introduced": "0"
+        }
+      ]
+    },
     "ecosystem_specific": {
-      "urgency": "unimportant"
+      "urgency": "end-of-life"
     }
   },
   {
     "pkg_name": "busybox",
-    "ecosystem": "Debian13",
-    "fixed_version": {},
+    "ecosystem": "Debian:11",
+    "fixed_version": {
+      "affected_versions": [
+        {
+          "introduced": "0"
+        }
+      ]
+    },
     "ecosystem_specific": {
       "urgency": "unimportant"
     }
   },
   {
     "pkg_name": "busybox",
-    "ecosystem": "Debian12",
-    "fixed_version": {},
+    "ecosystem": "Debian:12",
+    "fixed_version": {
+      "affected_versions": [
+        {
+          "introduced": "0"
+        }
+      ]
+    },
     "ecosystem_specific": {
       "urgency": "unimportant"
     }
   },
   {
     "pkg_name": "busybox",
-    "ecosystem": "Debian11",
-    "fixed_version": {},
+    "ecosystem": "Debian:13",
+    "fixed_version": {
+      "affected_versions": [
+        {
+          "introduced": "0"
+        }
+      ]
+    },
     "ecosystem_specific": {
       "urgency": "unimportant"
     }

vulnfeeds/tools/debian/debian_converter/first_package_finder.py

@@ -71,7 +71,9 @@ def retrieve_codename_to_version() -> pd.DataFrame:
     df = pd.read_csv(csv, dtype=str)
     # `series` appears to be `codename` but with no caps
     df['sources'] = ''
-    df.dropna(subset=['release'], inplace=True)
+    df.dropna(subset=['version'], inplace=True)
+    # Set `release` to `created` if not yet released
+    df['release'] = df['release'].fillna(df['created'])
     codename_to_version = df.set_index('series')
 
   return codename_to_version