perf(local): avoid loading the entire advisory unless it will actually be used (#2450)

Right now we always parse advisories even if we don't end up loading
them into the database, which is relatively expensive at scale so now we
use `gjson` to extract the subset of data we need from the raw bytes to
determine if the advisory is relevant before we do the advisory parsing.

This is especially useful for databases with a high amount of MAL
advisories since their packages are very rare, such as the NPM database
(which has 209647 MAL advisories out of a total of 214057 advisories) -
before this it takes about 10 seconds to do a scan, whereas after this
optimization it takes about 3 seconds

Author: G-Rath

internal/clients/clientimpl/localmatcher/zip.go

@@ -12,13 +12,15 @@ import (
 	"net/http"
 	"os"
 	"path"
+	"slices"
 	"strings"
 
 	"github.com/google/osv-scalibr/extractor"
 	"github.com/google/osv-scanner/v2/internal/cmdlogger"
 	"github.com/google/osv-scanner/v2/internal/imodels"
 	"github.com/google/osv-scanner/v2/internal/utility/vulns"
 	"github.com/ossf/osv-schema/bindings/go/osvschema"
+	"github.com/tidwall/gjson"
 	"google.golang.org/protobuf/encoding/protojson"
 )
 
@@ -160,16 +162,22 @@ func (db *ZipDB) fetchZip(ctx context.Context) (*os.File, error) {
 	return f, nil
 }
 
-func mightAffectPackages(v *osvschema.Vulnerability, names []string) bool {
-	for _, affected := range v.GetAffected() {
-		for _, name := range names {
-			if affected.GetPackage().GetName() == name {
-				return true
-			}
+func mightAffectPackagesBytes(content []byte, names []string) bool {
+	affected := gjson.GetBytes(content, "affected")
+
+	for _, name := range affected.Get("#.package.name").Array() {
+		if slices.Contains(names, name.String()) {
+			return true
+		}
+	}
 
-			// "name" will be the git repository in the case of the GIT ecosystem
-			for _, ran := range affected.GetRanges() {
-				if vulns.NormalizeRepo(ran.GetRepo()) == vulns.NormalizeRepo(name) {
+	for _, repos := range affected.Get("#.ranges.#.repo").Array() {
+		for _, repo := range repos.Array() {
+			repoName := vulns.NormalizeRepo(repo.String())
+
+			for _, name := range names {
+				// "name" will be the git repository in the case of the GIT ecosystem
+				if repoName == vulns.NormalizeRepo(name) {
 					return true
 				}
 			}
@@ -197,18 +205,20 @@ func (db *ZipDB) loadZipFile(zipFile *zip.File, names []string) {
 		return
 	}
 
+	// if we have been provided a list of package names, only load advisories
+	// that might actually affect those packages, rather than all advisories
+	if len(names) > 0 && !mightAffectPackagesBytes(content, names) {
+		return
+	}
+
 	vulnerability := &osvschema.Vulnerability{}
 	if err := protojson.Unmarshal(content, vulnerability); err != nil {
 		cmdlogger.Warnf("%s is not a valid JSON file: %v", zipFile.Name, err)
 
 		return
 	}
 
-	// if we have been provided a list of package names, only load advisories
-	// that might actually affect those packages, rather than all advisories
-	if len(names) == 0 || mightAffectPackages(vulnerability, names) {
-		db.Vulnerabilities = append(db.Vulnerabilities, vulnerability)
-	}
+	db.Vulnerabilities = append(db.Vulnerabilities, vulnerability)
 }
 
 // load fetches a zip archive of the OSV database and loads known vulnerabilities

internal/clients/clientimpl/localmatcher/zip_test.go

@@ -502,6 +502,34 @@ func TestNewZippedDB_WithSpecificPackages(t *testing.T) {
 					{Package: &osvschema.Package{Name: "pkg-2"}},
 				},
 			},
+			"GHSA-7.json": {
+				Id: "GHSA-7",
+				Affected: []*osvschema.Affected{
+					{
+						Ranges: []*osvschema.Range{
+							{Type: osvschema.Range_SEMVER},
+							{Type: osvschema.Range_GIT, Repo: "https://github.com/org/repo"},
+						},
+					},
+				},
+			},
+			"GHSA-8.json": {
+				Id: "GHSA-8",
+				Affected: []*osvschema.Affected{
+					{Ranges: []*osvschema.Range{{Type: osvschema.Range_SEMVER}}},
+					{Ranges: []*osvschema.Range{{Type: osvschema.Range_GIT, Repo: "git://github.com/org/repo.git"}}},
+				},
+			},
+			"GHSA-9.json": {
+				Id: "GHSA-9",
+				Affected: []*osvschema.Affected{
+					{
+						Ranges: []*osvschema.Range{
+							{Type: osvschema.Range_GIT, Repo: "https://github.com/anotherorg/anotherrepo"},
+						},
+					},
+				},
+			},
 		})
 	})
 
@@ -512,7 +540,7 @@ func TestNewZippedDB_WithSpecificPackages(t *testing.T) {
 		ts.URL,
 		userAgent,
 		false,
-		[]*extractor.Package{{Name: "pkg-1"}, {Name: "pkg-3"}},
+		[]*extractor.Package{{Name: "pkg-1"}, {Name: "pkg-3"}, {Name: "https://github.com/org/repo"}},
 	)
 
 	if err != nil {
@@ -545,5 +573,21 @@ func TestNewZippedDB_WithSpecificPackages(t *testing.T) {
 				{Package: &osvschema.Package{Name: "pkg-2"}},
 			},
 		},
+		{
+			Id: "GHSA-7",
+			Affected: []*osvschema.Affected{
+				{Ranges: []*osvschema.Range{
+					{Type: osvschema.Range_SEMVER},
+					{Type: osvschema.Range_GIT, Repo: "https://github.com/org/repo"},
+				}},
+			},
+		},
+		{
+			Id: "GHSA-8",
+			Affected: []*osvschema.Affected{
+				{Ranges: []*osvschema.Range{{Type: osvschema.Range_SEMVER}}},
+				{Ranges: []*osvschema.Range{{Type: osvschema.Range_GIT, Repo: "git://github.com/org/repo.git"}}},
+			},
+		},
 	})
 }