Skip to content

feat: deduplicate API calls for identical PURLs #132

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
snyk-will merged 1 commit into from
Feb 9, 2026
+216 −35
Merged

feat: deduplicate API calls for identical PURLs #132

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 46 additions & 19 deletions lib/snyk/enrich_cyclonedx.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,11 @@ import (

type cdxEnricher = func(*Config, *cdx.Component, *packageurl.PackageURL)

type cdxPurlGroup struct {
purl packageurl.PackageURL
components []*cdx.Component
}

var cdxEnrichers = []cdxEnricher{
enrichCDXSnykAdvisorData,
enrichCDXSnykVulnerabilityDBData,
Expand Down Expand Up @@ -85,32 +90,52 @@ func enrichCycloneDX(cfg *Config, bom *cdx.BOM, logger *zerolog.Logger) *cdx.BOM
logger.Debug().Str("org_id", orgID.String()).Msg("Inferred Snyk organization ID")

var mutex = &sync.Mutex{}
vulnerabilities := make(map[cdx.Component][]issues.CommonIssueModelVThree)
vulnerabilities := make(map[*cdx.Component][]issues.CommonIssueModelVThree)
wg := sizedwaitgroup.New(20)

comps := utils.DiscoverCDXComponents(bom)
logger.Debug().Msgf("Detected %d packages", len(comps))

// Group components by PURL to deduplicate API calls
purlGroups := make(map[string]*cdxPurlGroup)
for i := range comps {
component := comps[i]
l := logger.With().Str("bom-ref", component.BOMRef).Logger()

purl, err := packageurl.FromString(component.PackageURL)
if err != nil {
l.Debug().
Err(err).
Msg("Could not identify package")
continue
}
for _, enrichFunc := range cdxEnrichers {
enrichFunc(cfg, component, &purl)
}

key := purl.ToString()
group, ok := purlGroups[key]
Comment on lines +116 to +117
Copy link
Collaborator

@mcombuechen mcombuechen Feb 9, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe a bit cheaper not to serialize again, but use the original purl string:

Suggested change
key := purl.ToString()
group, ok := purlGroups[key]
group, ok := purlGroups[component.PackageURL]

if !ok {
group = &cdxPurlGroup{purl: purl}
purlGroups[key] = group
}
group.components = append(group.components, component)
}

logger.Debug().Msgf("Detected %d unique PURLs", len(purlGroups))

// Fetch vulnerabilities for each unique PURL
for _, group := range purlGroups {
wg.Add()
go func(component *cdx.Component) {
go func() {
defer wg.Done()
l := logger.With().Str("bom-ref", component.BOMRef).Logger()
l := logger.With().
Str("purl", group.purl.ToString()).
Logger()

purl, err := packageurl.FromString(component.PackageURL)
if err != nil {
l.Debug().
Err(err).
Msg("Could not identify package")
return
}
for _, enrichFunc := range cdxEnrichers {
enrichFunc(cfg, component, &purl)
}
resp, err := GetPackageVulnerabilities(cfg, &purl, auth, orgID, logger)
resp, err := GetPackageVulnerabilities(cfg, &group.purl, auth, orgID, logger)
if err != nil {
l.Err(err).
Str("purl", purl.ToString()).
Msg("Failed to fetch vulnerabilities for package")
return
}
Expand All @@ -126,18 +151,20 @@ func enrichCycloneDX(cfg *Config, bom *cdx.BOM, logger *zerolog.Logger) *cdx.BOM

if packageDoc.Data != nil {
mutex.Lock()
vulnerabilities[*component] = *packageDoc.Data
for _, component := range group.components {
vulnerabilities[component] = *packageDoc.Data
}
mutex.Unlock()
}
}(comps[i])
}()
}
wg.Wait()

var vulns []cdx.Vulnerability
for k, v := range vulnerabilities {
for comp, v := range vulnerabilities {
for _, issue := range v {
vuln := cdx.Vulnerability{
BOMRef: k.BOMRef,
BOMRef: comp.BOMRef,
}
if issue.Id != nil {
vuln.ID = *issue.Id
Expand Down
57 changes: 41 additions & 16 deletions lib/snyk/enrich_spdx.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,11 @@ import (

type spdxEnricher = func(*Config, *spdx_2_3.Package, *packageurl.PackageURL)

type spdxPurlGroup struct {
purl packageurl.PackageURL
packages []*spdx_2_3.Package
}

var spdxEnrichers = []spdxEnricher{
enrichSPDXSnykAdvisorData,
enrichSPDXSnykVulnerabilityDBData,
Expand Down Expand Up @@ -97,25 +102,43 @@ func enrichSPDX(cfg *Config, bom *spdx.Document, logger *zerolog.Logger) *spdx.D
packages := bom.Packages
logger.Debug().Msgf("Detected %d packages", len(packages))

for i, pkg := range packages {
wg.Add()
// Group packages by PURL to deduplicate API calls
purlGroups := make(map[string]*spdxPurlGroup)
for _, pkg := range packages {
l := logger.With().Str("SPDXID", string(pkg.PackageSPDXIdentifier)).Logger()

purl, err := utils.GetPurlFromSPDXPackage(pkg)
if err != nil || purl == nil {
l.Debug().Msg("Could not identify package")
continue
}
for _, enrichFn := range spdxEnrichers {
enrichFn(cfg, pkg, purl)
}

key := purl.ToString()
group, ok := purlGroups[key]
if !ok {
group = &spdxPurlGroup{purl: *purl}
purlGroups[key] = group
}
group.packages = append(group.packages, pkg)
}

go func(pkg *spdx_2_3.Package, i int) {
logger.Debug().Msgf("Detected %d unique PURLs", len(purlGroups))

// Fetch vulnerabilities for each unique PURL
for _, group := range purlGroups {
wg.Add()
go func() {
defer wg.Done()
l := logger.With().Str("SPDXID", string(pkg.PackageSPDXIdentifier)).Logger()
l := logger.With().
Str("purl", group.purl.ToString()).
Logger()

purl, err := utils.GetPurlFromSPDXPackage(pkg)
if err != nil || purl == nil {
l.Debug().Msg("Could not identify package")
return
}
for _, enrichFn := range spdxEnrichers {
enrichFn(cfg, pkg, purl)
}
resp, err := GetPackageVulnerabilities(cfg, purl, auth, orgID, logger)
resp, err := GetPackageVulnerabilities(cfg, &group.purl, auth, orgID, logger)
if err != nil {
l.Err(err).
Str("purl", purl.ToString()).
Msg("Failed to fetch vulnerabilities for package")
return
}
Expand All @@ -131,10 +154,12 @@ func enrichSPDX(cfg *Config, bom *spdx.Document, logger *zerolog.Logger) *spdx.D

if packageDoc.Data != nil {
mutex.Lock()
vulnerabilities[pkg] = *packageDoc.Data
for _, pkg := range group.packages {
vulnerabilities[pkg] = *packageDoc.Data
}
mutex.Unlock()
}
}(pkg, i)
}()
}

wg.Wait()
Expand Down
129 changes: 129 additions & 0 deletions lib/snyk/enrich_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
_ "embed"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"

cdx "github.com/CycloneDX/cyclonedx-go"
Expand Down Expand Up @@ -56,6 +57,61 @@ func TestEnrichSBOM_CycloneDXWithVulnerabilities(t *testing.T) {
assert.Equal(t, (*vuln.Ratings)[1].Method, cdx.ScoringMethodCVSSv3)
}

func TestEnrichSBOM_CycloneDXDeduplicatesRequests(t *testing.T) {
var numRequests int32
mux := http.NewServeMux()
mux.HandleFunc(
"GET /rest/self",
func(w http.ResponseWriter, r *http.Request) {
respond(w, selfBody)
})
mux.HandleFunc(
"GET /rest/orgs/{org_id}/packages/{purl}/issues",
func(w http.ResponseWriter, r *http.Request) {
atomic.AddInt32(&numRequests, 1)
respond(w, numpyIssues)
})

srv := httptest.NewServer(mux)
t.Cleanup(srv.Close)

cfg := DefaultConfig()
cfg.APIToken = "asdf"
cfg.SnykAPIURL = srv.URL

logger := zerolog.Nop()
svc := NewService(cfg, &logger)

bom := &cdx.BOM{
Components: &[]cdx.Component{
{
BOMRef: "pkg:pypi/numpy@1.16.0",
Name: "numpy",
Version: "1.16.0",
PackageURL: "pkg:pypi/numpy@1.16.0",
},
{
BOMRef: "pkg:pypi/numpy@1.16.0#dup",
Name: "numpy",
Version: "1.16.0",
PackageURL: "pkg:pypi/numpy@1.16.0",
},
},
}
doc := &sbom.SBOMDocument{BOM: bom}

svc.EnrichSBOM(doc)

assert.Equal(t, int32(1), atomic.LoadInt32(&numRequests))
require.NotNil(t, bom.Vulnerabilities)
vulnByRef := map[string]int{}
for _, vuln := range *bom.Vulnerabilities {
vulnByRef[vuln.BOMRef]++
}
assert.Greater(t, vulnByRef["pkg:pypi/numpy@1.16.0"], 0)
assert.Greater(t, vulnByRef["pkg:pypi/numpy@1.16.0#dup"], 0)
}

func TestEnrichSBOM_CycloneDXExternalRefs(t *testing.T) {
svc := setupTestEnv(t)

Expand Down Expand Up @@ -199,6 +255,79 @@ func TestEnrichSBOM_SPDXWithVulnerabilities(t *testing.T) {
assert.Equal(t, "Arbitrary Code Execution", vulnRef.ExternalRefComment)
}

func TestEnrichSBOM_SPDXDeduplicatesRequests(t *testing.T) {
var numRequests int32
mux := http.NewServeMux()
mux.HandleFunc(
"GET /rest/self",
func(w http.ResponseWriter, r *http.Request) {
respond(w, selfBody)
})
mux.HandleFunc(
"GET /rest/orgs/{org_id}/packages/{purl}/issues",
func(w http.ResponseWriter, r *http.Request) {
atomic.AddInt32(&numRequests, 1)
respond(w, numpyIssues)
})

srv := httptest.NewServer(mux)
t.Cleanup(srv.Close)

cfg := DefaultConfig()
cfg.APIToken = "asdf"
cfg.SnykAPIURL = srv.URL

logger := zerolog.Nop()
svc := NewService(cfg, &logger)

bom := &spdx_2_3.Document{
Packages: []*spdx_2_3.Package{
{
PackageSPDXIdentifier: "pkg:pypi/numpy@1.16.0",
PackageName: "numpy",
PackageVersion: "1.16.0",
PackageExternalReferences: []*spdx_2_3.PackageExternalReference{
{
Category: spdx.CategoryPackageManager,
RefType: "purl",
Locator: "pkg:pypi/numpy@1.16.0",
},
},
},
{
PackageSPDXIdentifier: "pkg:pypi/numpy@1.16.0-dup",
PackageName: "numpy",
PackageVersion: "1.16.0",
PackageExternalReferences: []*spdx_2_3.PackageExternalReference{
{
Category: spdx.CategoryPackageManager,
RefType: "purl",
Locator: "pkg:pypi/numpy@1.16.0",
},
},
},
},
}
doc := &sbom.SBOMDocument{BOM: bom}

svc.EnrichSBOM(doc)

assert.Equal(t, int32(1), atomic.LoadInt32(&numRequests))
expectedLocator := "https://security.snyk.io/vuln/SNYK-PYTHON-NUMPY-73513"
for _, pkg := range bom.Packages {
hasVulnRef := false
for _, ref := range pkg.PackageExternalReferences {
if ref.Category == spdx.CategorySecurity &&
ref.RefType == "advisory" &&
ref.Locator == expectedLocator {
hasVulnRef = true
break
}
}
assert.Truef(t, hasVulnRef, "expected vulnerability reference for %s", pkg.PackageSPDXIdentifier)
}
}

func TestEnrichSBOM_SPDXExternalRefs(t *testing.T) {
svc := setupTestEnv(t)

Expand Down
Loading