feat(vulnfeeds): Enable git resolution for affected versions field CVEList

vulnfeeds/cmd/cve-bulk-converter/main.go

@@ -21,7 +21,7 @@ var (
 	localOutputDir = flag.String("out_dir", "cvelist2osv", "Path to output results.")
 	years          = flag.String("years", "2022,2023,2024,2025", "A comma-separated list of years to process.")
 	workers        = flag.Int("workers", 30, "The number of concurrent workers to use for processing CVEs.")
-	cnas           = flag.String("cnas", "Linux", "A comma-separated list of CNAs to process.")
+	cnas           = flag.String("cnas", "Linux,GitHub_M", "A comma-separated list of CNAs to process.")
 )
 
 func main() {

vulnfeeds/cmd/cvelist2osv/converter.go

@@ -79,21 +79,25 @@ func FromCVE5(cve cves.CVE5, refs []cves.Reference, metrics *ConversionMetrics)
 
 	published, err := cves.ParseCVE5Timestamp(cve.Metadata.DatePublished)
 	if err != nil {
-		metrics.Notes = append(metrics.Notes, "Published date failed to parse, setting time to now")
+		metrics.Notes = append(metrics.Notes, fmt.Sprintf("[%s]: Published date failed to parse, setting time to now", cve.Metadata.CVEID))
 		published = time.Now()
 	}
 	v.Published = published
 
 	modified, err := cves.ParseCVE5Timestamp(cve.Metadata.DateUpdated)
 	if err != nil {
-		metrics.Notes = append(metrics.Notes, "Modified date failed to parse, setting time to now")
+		metrics.Notes = append(metrics.Notes, fmt.Sprintf("[%s]: Modified date failed to parse, setting time to now", cve.Metadata.CVEID))
 		modified = time.Now()
 	}
 	v.Modified = modified
 
-	// Add affected version information.
-	AddVersionInfo(cve, &v, metrics)
+	// Try to extract repository URLs from references.
+	repos, repoNotes := cves.ReposFromReferencesCVEList(string(cve.Metadata.CVEID), refs, RefTagDenyList)
+	metrics.Notes = append(metrics.Notes, repoNotes...)
+	metrics.Repos = repos
 
+	// Add affected version information.
+	AddVersionInfo(cve, &v, metrics, repos)
 	// TODO(jesslowe@): Add CWEs.
 
 	// Combine severity metrics from both CNA and ADP containers.
@@ -166,17 +170,12 @@ func ConvertAndExportCVEToOSV(cve cves.CVE5, directory string) error {
 	cveID := cve.Metadata.CVEID
 	cnaAssigner := cve.Metadata.AssignerShortName
 	references := identifyPossibleURLs(cve)
-	metrics := &ConversionMetrics{}
+	metrics := ConversionMetrics{}
 	// Create a base OSV record from the CVE.
-	v := FromCVE5(cve, references, metrics)
+	v := FromCVE5(cve, references, &metrics)
 
 	// Collect metrics about the conversion.
-	extractConversionMetrics(cve, v.References, metrics)
-
-	// Try to extract repository URLs from references.
-	repos, repoNotes := cves.ReposFromReferencesCVEList(string(cveID), references, RefTagDenyList)
-	metrics.Notes = append(metrics.Notes, repoNotes...)
-	metrics.Repos = repos
+	extractConversionMetrics(cve, v.References, &metrics)
 
 	vulnDir := filepath.Join(directory, cnaAssigner)
 
@@ -186,7 +185,7 @@ func ConvertAndExportCVEToOSV(cve cves.CVE5, directory string) error {
 	}
 
 	// Save the conversion metrics to a file.
-	if err := writeMetricToFile(cveID, vulnDir, metrics); err != nil {
+	if err := writeMetricToFile(cveID, vulnDir, &metrics); err != nil {
 		return err
 	}
 

vulnfeeds/cmd/cvelist2osv/converter_test.go

@@ -191,13 +191,18 @@ func TestFromCVE5(t *testing.T) {
 						},
 					},
 					Affected: []osvschema.Affected{{
-						// DatabaseSpecific: map[string]interface{}{
-						// 	"CPE": []string{"cpe:2.3:a:gitlab:gitlab:*:*:*:*:*:*:*:*"},
-						// },
-
-						Ranges: []osvschema.Range{{Type: "ECOSYSTEM",
-							Events: []osvschema.Event{{Introduced: "18.0"}, {Fixed: "18.0.1"}},
-						}}}},
+						Ranges: []osvschema.Range{{
+							Type: "GIT",
+							Repo: "https://gitlab.com/gitlab-org/gitlab",
+							Events: []osvschema.Event{
+								{Introduced: "504fd9e5236e13d674e051c6b8a1e9892b371c58"},
+								{Fixed: "3426be1b93852c5358240c5df40970c0ddfbdb2a"},
+							},
+							DatabaseSpecific: map[string]any{
+								"versions": []osvschema.Event{{Introduced: "18.0"}, {Fixed: "18.0.1"}},
+							},
+						}},
+					}},
 				},
 			},
 		},
@@ -226,9 +231,19 @@ func TestFromCVE5(t *testing.T) {
 							Score: "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H",
 						},
 					},
-					Affected: []osvschema.Affected{{Ranges: []osvschema.Range{{
-						Type:   "ECOSYSTEM",
-						Events: []osvschema.Event{{Introduced: "0"}, {Fixed: "1.10.5"}}}}}},
+					Affected: []osvschema.Affected{{
+						Ranges: []osvschema.Range{{
+							Type: "GIT",
+							Repo: "https://github.com/amazon-ion/ion-java",
+							Events: []osvschema.Event{
+								{Introduced: "0"},
+								{Fixed: "019a6117fb99131f74f92ecf462169613234abbf"},
+							},
+							DatabaseSpecific: map[string]any{
+								"versions": []osvschema.Event{{Introduced: "0"}, {Fixed: "1.10.5"}},
+							},
+						}},
+					}},
 					DatabaseSpecific: nil,
 				},
 			},

vulnfeeds/cmd/cvelist2osv/version_extraction.go

@@ -4,13 +4,15 @@ import (
 	"cmp"
 	"errors"
 	"fmt"
+	"log/slog"
 	"strconv"
 	"strings"
 
 	"slices"
 
 	"github.com/google/osv/vulnfeeds/cves"
 	"github.com/google/osv/vulnfeeds/git"
+	"github.com/google/osv/vulnfeeds/utility/logger"
 	"github.com/google/osv/vulnfeeds/vulns"
 	"github.com/ossf/osv-schema/bindings/go/osvschema"
 )
@@ -72,7 +74,7 @@ func toVersionRangeType(s string) VersionRangeType {
 // 3. If no versions are found, it falls back to searching for CPEs in the CNA container.
 // 4. As a last resort, it attempts to extract version information from the description text (currently not saved).
 // It returns the source of the version information and a slice of notes detailing the extraction process.
-func AddVersionInfo(cve cves.CVE5, v *vulns.Vulnerability, metrics *ConversionMetrics) {
+func AddVersionInfo(cve cves.CVE5, v *vulns.Vulnerability, metrics *ConversionMetrics, repos []string) {
 	gotVersions := false
 
 	// Combine 'affected' entries from both CNA and ADP containers.
@@ -104,31 +106,36 @@ func AddVersionInfo(cve cves.CVE5, v *vulns.Vulnerability, metrics *ConversionMe
 			hasGit = true
 		}
 
-		aff := osvschema.Affected{}
-		for _, vr := range versionRanges {
-			if versionType == VersionRangeTypeGit {
-				vr.Type = osvschema.RangeGit
-				vr.Repo = cveAff.Repo
-			} else {
-				vr.Type = osvschema.RangeEcosystem
-			}
-			aff.Ranges = append(aff.Ranges, vr)
-		}
-
+		var aff osvschema.Affected
 		// Special handling for Linux kernel CVEs.
-		if cve.Metadata.AssignerShortName == "Linux" && versionType != VersionRangeTypeGit {
-			aff.Package = osvschema.Package{
-				Ecosystem: string(osvschema.EcosystemLinux),
-				Name:      "Kernel",
+		if cve.Metadata.AssignerShortName == "Linux" {
+			for _, vr := range versionRanges {
+				if versionType == VersionRangeTypeGit {
+					vr.Type = osvschema.RangeGit
+					vr.Repo = cveAff.Repo
+				} else {
+					vr.Type = osvschema.RangeEcosystem
+				}
+				aff.Ranges = append(aff.Ranges, vr)
+			}
+			if versionType != VersionRangeTypeGit {
+				aff.Package = osvschema.Package{
+					Ecosystem: string(osvschema.EcosystemLinux),
+					Name:      "Kernel",
+				}
+			}
+		} else {
+			var err error
+			aff, err = gitVersionsToCommits(cve.Metadata.CVEID, versionRanges, repos, make(git.RepoTagsCache))
+			if err != nil {
+				logger.Error("Failed to convert git versions to commits", slog.Any("err", err))
+			} else {
+				hasGit = true
 			}
 		}
 
 		v.Affected = append(v.Affected, aff)
-		if hasGit {
-			metrics.VersionSources = append(metrics.VersionSources, VersionSourceGit)
-		} else {
-			metrics.VersionSources = append(metrics.VersionSources, VersionSourceAffected)
-		}
+		metrics.VersionSources = append(metrics.VersionSources, VersionSourceAffected)
 	}
 
 	// If no versions were found so far, fall back to CPEs.
@@ -164,6 +171,104 @@ func AddVersionInfo(cve cves.CVE5, v *vulns.Vulnerability, metrics *ConversionMe
 	}
 }
 
+// Examines repos and tries to convert versions to commits by treating them as Git tags.
+// Takes a CVE ID string (for logging), VersionInfo with AffectedVersions and
+// typically no AffectedCommits and attempts to add AffectedCommits (including Fixed commits) where there aren't any.
+// Refuses to add the same commit to AffectedCommits more than once.
+func gitVersionsToCommits(cveID cves.CVEID, versionRanges []osvschema.Range, repos []string, cache git.RepoTagsCache) (osvschema.Affected, error) {
+	var newAff osvschema.Affected
+	var newVersionRanges []osvschema.Range
+	unresolvedRanges := versionRanges
+
+	for _, repo := range repos {
+		if len(unresolvedRanges) == 0 {
+			break // All ranges have been resolved.
+		}
+
+		normalizedTags, err := git.NormalizeRepoTags(repo, cache)
+		if err != nil {
+			logger.Warn("Failed to normalize tags", slog.String("cve", string(cveID)), slog.String("repo", repo), slog.Any("err", err))
+			continue
+		}
+
+		var stillUnresolvedRanges []osvschema.Range
+		for _, vr := range unresolvedRanges {
+			var introducedCommit, fixedCommit, lastAffectedCommit string
+			var resolutionErr error
+
+			for _, ev := range vr.Events {
+				logger.Info("Attempting version resolution", slog.String("cve", string(cveID)), slog.Any("event", ev), slog.String("repo", repo))
+				if ev.Introduced != "" {
+					if ev.Introduced == "0" {
+						introducedCommit = "0"
+					} else {
+						introducedCommit, resolutionErr = git.VersionToCommit(ev.Introduced, normalizedTags)
+						if resolutionErr != nil {
+							logger.Warn("Failed to get Git commit for introduced version", slog.String("cve", string(cveID)), slog.String("version", ev.Introduced), slog.String("repo", repo), slog.Any("err", resolutionErr))
+						} else {
+							logger.Info("Successfully derived commit for introduced version", slog.String("cve", string(cveID)), slog.String("commit", introducedCommit), slog.String("version", ev.Introduced))
+						}
+					}
+				}
+				if ev.Fixed != "" {
+					fixedCommit, resolutionErr = git.VersionToCommit(ev.Fixed, normalizedTags)
+					if resolutionErr != nil {
+						logger.Warn("Failed to get Git commit for fixed version", slog.String("cve", string(cveID)), slog.String("version", ev.Fixed), slog.String("repo", repo), slog.Any("err", resolutionErr))
+					} else {
+						logger.Info("Successfully derived commit for fixed version", slog.String("cve", string(cveID)), slog.String("commit", fixedCommit), slog.String("version", ev.Fixed))
+					}
+				}
+				if ev.LastAffected != "" {
+					lastAffectedCommit, resolutionErr = git.VersionToCommit(ev.LastAffected, normalizedTags)
+					if resolutionErr != nil {
+						logger.Warn("Failed to get Git commit for last affected version", slog.String("cve", string(cveID)), slog.String("version", ev.LastAffected), slog.String("repo", repo), slog.Any("err", resolutionErr))
+					} else {
+						logger.Info("Successfully derived commit for last affected version", slog.String("cve", string(cveID)), slog.String("commit", lastAffectedCommit), slog.String("version", ev.LastAffected))
+					}
+				}
+			}
+
+			resolved := false
+			if fixedCommit != "" && introducedCommit != "" {
+				newVR := buildVersionRange(introducedCommit, "", fixedCommit)
+				newVR.Repo = repo
+				newVR.Type = osvschema.RangeGit
+				newVR.DatabaseSpecific = make(map[string]any)
+				newVR.DatabaseSpecific["versions"] = vr.Events
+				newVersionRanges = append(newVersionRanges, newVR)
+				resolved = true
+			} else if lastAffectedCommit != "" && introducedCommit != "" {
+				newVR := buildVersionRange(introducedCommit, lastAffectedCommit, "")
+				newVR.Repo = repo
+				newVR.Type = osvschema.RangeGit
+				newVR.DatabaseSpecific = make(map[string]any)
+				newVR.DatabaseSpecific["versions"] = vr.Events
+				newVersionRanges = append(newVersionRanges, newVR)
+				resolved = true
+			}
+
+			if !resolved {
+				stillUnresolvedRanges = append(stillUnresolvedRanges, vr)
+			}
+		}
+		unresolvedRanges = stillUnresolvedRanges
+	}
+
+	var err error
+	if len(unresolvedRanges) > 0 {
+		newAff.DatabaseSpecific = make(map[string]any)
+		newAff.DatabaseSpecific["unresolved_versions"] = unresolvedRanges
+	}
+
+	if len(newVersionRanges) > 0 {
+		newAff.Ranges = newVersionRanges
+	} else if len(unresolvedRanges) > 0 { // Only error if there were ranges to resolve but none were.
+		err = errors.New("was not able to get git version ranges")
+	}
+
+	return newAff, err
+}
+
 // findCPEVersionRanges extracts version ranges and CPE strings from the CNA's
 // CPE applicability statements in a CVE record.
 func findCPEVersionRanges(cve cves.CVE5) (versionRanges []osvschema.Range, cpes []string, err error) {
@@ -343,7 +448,6 @@ func findNormalAffectedRanges(affected cves.Affected, metrics *ConversionMetrics
 		// affected, but more likely, it affects up to that version. It could also mean that the range is given
 		// in one line instead - like "< 1.5.3" or "< 2.45.4, >= 2.0 " or just "before 1.4.7", so check for that.
 		metrics.Notes = append(metrics.Notes, "Only version exists")
-		// GitHub often encodes the range directly in the version string.
 
 		av, err := git.ParseVersionRange(vers.Version)
 		if err == nil {