perf(local): only load advisories that are about the packages being scanned (#2241)

This updates the local matcher to have it skip advisories that don't
have at least one affected entry with a package name matching one of the
packages being scanned in the current run, which can greatly reduce the
peak memory usage for databases like Ubuntu (going from something like
10gb down to 1gb).

Since we cache databases based on their ecosystem only, this does mean
subsequent calls to `LocalMatcher#MatchVulnerabilities` will not give
any results for packages that were not present in the first call - while
this shouldn't be a problem currently since we handle creating the
`VulnerabilityMatcher` as part of scanning, I've added a basic guard
that returns an error if the function is called with any "partial"
database cached to catch this (be it on purpose or because of a bug)

This should not impact guided remediation since it explicitly loads the
database before doing any work meaning this change won't help it but
should also not hurt it

Resolves #2217 (again)

Author: G-Rath

internal/clients/clientimpl/localmatcher/localmatcher.go

@@ -49,6 +49,15 @@ func NewLocalMatcher(localDBPath string, userAgent string, downloadDB bool) (*Lo
 func (matcher *LocalMatcher) MatchVulnerabilities(ctx context.Context, invs []*extractor.Package) ([][]*osvschema.Vulnerability, error) {
 	results := make([][]*osvschema.Vulnerability, 0, len(invs))
 
+	// ensure all databases loaded so far have been fully loaded; this is just a
+	// basic safeguard since we don't actually currently attempt to reuse matchers
+	// across scans, and its possible we never will, so we don't need to be smart
+	for _, db := range matcher.dbs {
+		if db.Partial {
+			return nil, errors.New("local matcher cannot be (re)used with a partially loaded database")
+		}
+	}
+
 	for _, inv := range invs {
 		if ctx.Err() != nil {
 			return nil, ctx.Err()
@@ -77,7 +86,7 @@ func (matcher *LocalMatcher) MatchVulnerabilities(ctx context.Context, invs []*e
 			eco = "GIT"
 		}
 
-		db, err := matcher.loadDBFromCache(ctx, eco)
+		db, err := matcher.loadDBFromCache(ctx, eco, invs)
 
 		if err != nil {
 			// no logging here as the loader will have already done that
@@ -94,13 +103,15 @@ func (matcher *LocalMatcher) MatchVulnerabilities(ctx context.Context, invs []*e
 
 // LoadEcosystem tries to preload the ecosystem into the cache, and returns an error if the ecosystem
 // cannot be loaded.
+//
+// Preloaded databases include every advisory, so can be reused.
 func (matcher *LocalMatcher) LoadEcosystem(ctx context.Context, eco osvecosystem.Parsed) error {
-	_, err := matcher.loadDBFromCache(ctx, eco.Ecosystem)
+	_, err := matcher.loadDBFromCache(ctx, eco.Ecosystem, nil)
 
 	return err
 }
 
-func (matcher *LocalMatcher) loadDBFromCache(ctx context.Context, eco osvschema.Ecosystem) (*ZipDB, error) {
+func (matcher *LocalMatcher) loadDBFromCache(ctx context.Context, eco osvschema.Ecosystem, invs []*extractor.Package) (*ZipDB, error) {
 	if db, ok := matcher.dbs[eco]; ok {
 		return db, nil
 	}
@@ -109,7 +120,15 @@ func (matcher *LocalMatcher) loadDBFromCache(ctx context.Context, eco osvschema.
 		return nil, matcher.failedDBs[eco]
 	}
 
-	db, err := NewZippedDB(ctx, matcher.dbBasePath, string(eco), fmt.Sprintf("%s/%s/all.zip", zippedDBRemoteHost, eco), matcher.userAgent, !matcher.downloadDB)
+	db, err := NewZippedDB(
+		ctx,
+		matcher.dbBasePath,
+		string(eco),
+		fmt.Sprintf("%s/%s/all.zip", zippedDBRemoteHost, eco),
+		matcher.userAgent,
+		!matcher.downloadDB,
+		invs,
+	)
 
 	if err != nil {
 		matcher.failedDBs[eco] = err

internal/clients/clientimpl/localmatcher/zip.go

@@ -16,6 +16,7 @@ import (
 	"path"
 	"strings"
 
+	"github.com/google/osv-scalibr/extractor"
 	"github.com/google/osv-scanner/v2/internal/cmdlogger"
 	"github.com/google/osv-scanner/v2/internal/imodels"
 	"github.com/google/osv-scanner/v2/internal/utility/vulns"
@@ -35,6 +36,10 @@ type ZipDB struct {
 	Vulnerabilities []osvschema.Vulnerability
 	// User agent to query with
 	UserAgent string
+
+	// whether this database only has some of the advisories
+	// loaded from the underlying zip file
+	Partial bool
 }
 
 var ErrOfflineDatabaseNotFound = errors.New("no offline version of the OSV database is available")
@@ -143,9 +148,28 @@ func (db *ZipDB) fetchZip(ctx context.Context) ([]byte, error) {
 	return body, nil
 }
 
+func mightAffectPackages(v osvschema.Vulnerability, names []string) bool {
+	for _, affected := range v.Affected {
+		for _, name := range names {
+			if affected.Package.Name == name {
+				return true
+			}
+
+			// "name" will be the git repository in the case of the GIT ecosystem
+			for _, ran := range affected.Ranges {
+				if ran.Repo == name {
+					return true
+				}
+			}
+		}
+	}
+
+	return false
+}
+
 // Loads the given zip file into the database as an OSV.
 // It is assumed that the file is JSON and in the working directory of the db
-func (db *ZipDB) loadZipFile(zipFile *zip.File) {
+func (db *ZipDB) loadZipFile(zipFile *zip.File, names []string) {
 	file, err := zipFile.Open()
 	if err != nil {
 		cmdlogger.Warnf("Could not read %s: %v", zipFile.Name, err)
@@ -169,16 +193,23 @@ func (db *ZipDB) loadZipFile(zipFile *zip.File) {
 		return
 	}
 
-	db.Vulnerabilities = append(db.Vulnerabilities, vulnerability)
+	// if we have been provided a list of package names, only load advisories
+	// that might actually affect those packages, rather than all advisories
+	if len(names) == 0 || mightAffectPackages(vulnerability, names) {
+		db.Vulnerabilities = append(db.Vulnerabilities, vulnerability)
+	}
 }
 
 // load fetches a zip archive of the OSV database and loads known vulnerabilities
 // from it (which are assumed to be in json files following the OSV spec).
 //
+// If a list of package names is provided, then only advisories with at least
+// one affected entry for a listed package will be loaded.
+//
 // Internally, the archive is cached along with the date that it was fetched
 // so that a new version of the archive is only downloaded if it has been
 // modified, per HTTP caching standards.
-func (db *ZipDB) load(ctx context.Context) error {
+func (db *ZipDB) load(ctx context.Context, names []string) error {
 	db.Vulnerabilities = []osvschema.Vulnerability{}
 
 	body, err := db.fetchZip(ctx)
@@ -198,21 +229,33 @@ func (db *ZipDB) load(ctx context.Context) error {
 			continue
 		}
 
-		db.loadZipFile(zipFile)
+		db.loadZipFile(zipFile, names)
 	}
 
 	return nil
 }
 
-func NewZippedDB(ctx context.Context, dbBasePath, name, url, userAgent string, offline bool) (*ZipDB, error) {
+func NewZippedDB(ctx context.Context, dbBasePath, name, url, userAgent string, offline bool, invs []*extractor.Package) (*ZipDB, error) {
 	db := &ZipDB{
 		Name:       name,
 		ArchiveURL: url,
 		Offline:    offline,
 		StoredAt:   path.Join(dbBasePath, name, "all.zip"),
 		UserAgent:  userAgent,
+
+		// we only fully load the database if we're not provided a list of packages
+		Partial: len(invs) != 0,
 	}
-	if err := db.load(ctx); err != nil {
+	names := make([]string, 0, len(invs))
+
+	// map the packages to their names ahead of loading,
+	// to make things simpler and reduce double working
+	for _, inv := range invs {
+		in := imodels.FromInventory(inv)
+		names = append(names, in.Name())
+	}
+
+	if err := db.load(ctx, names); err != nil {
 		return nil, fmt.Errorf("unable to fetch OSV database: %w", err)
 	}