Update packages and hide generic packages that are already covered

Author: cdupuis

Dockerfile

@@ -1,5 +1,5 @@
 # build stage
-FROM golang:1.19-alpine3.16@sha256:0eb08c89ab1b0c638a9fe2780f7ae3ab18f6ecda2c76b908e09eb8073912045d as build
+FROM golang:1.20-alpine3.17 as build
 
 RUN apk add --no-cache git build-base ca-certificates
 

go.mod

@@ -1,12 +1,12 @@
 module github.com/docker/index-cli-plugin
 
-go 1.19
+go 1.20
 
 require (
 	github.com/Microsoft/go-winio v0.6.0
 	github.com/anchore/packageurl-go v0.1.1-0.20230104203445-02e0a6721501
-	github.com/anchore/stereoscope v0.0.0-20230222185948-fab1c9638abc
-	github.com/anchore/syft v0.73.0
+	github.com/anchore/stereoscope v0.0.0-20230301191755-abfb374a1122
+	github.com/anchore/syft v0.74.1
 	github.com/aquasecurity/trivy v0.30.4
 	github.com/atomist-skills/go-skill v0.0.6-0.20221221214636-a7de163fd901
 	github.com/briandowns/spinner v1.12.0
@@ -161,11 +161,11 @@ require (
 	github.com/shogo82148/go-shuffle v0.0.0-20170808115208-59829097ff3b // indirect
 	github.com/shopspring/decimal v1.2.0 // indirect
 	github.com/spdx/tools-golang v0.5.0-rc1 // indirect
-	github.com/spf13/afero v1.9.3 // indirect
+	github.com/spf13/afero v1.9.5 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/objx v0.5.0 // indirect
-	github.com/stretchr/testify v1.8.1 // indirect
+	github.com/stretchr/testify v1.8.2 // indirect
 	github.com/sylabs/sif/v2 v2.8.1 // indirect
 	github.com/sylabs/squashfs v0.6.1 // indirect
 	github.com/therootcompany/xz v1.0.1 // indirect
@@ -175,7 +175,7 @@ require (
 	github.com/vbatts/tar-split v0.11.2 // indirect
 	github.com/vifraa/gopom v0.2.1 // indirect
 	github.com/wagoodman/go-partybus v0.0.0-20210627031916-db1f5573bbc5 // indirect
-	github.com/wagoodman/go-progress v0.0.0-20200731105512-1020f39e6240 // indirect
+	github.com/wagoodman/go-progress v0.0.0-20230301185719-21920a456ad5 // indirect
 	github.com/xanzy/ssh-agent v0.3.0 // indirect
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
 	github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
@@ -186,14 +186,14 @@ require (
 	go.uber.org/zap v1.23.0 // indirect
 	golang.org/x/crypto v0.3.0 // indirect
 	golang.org/x/exp v0.0.0-20230202163644-54bba9f4231b // indirect
-	golang.org/x/mod v0.8.0 // indirect
-	golang.org/x/net v0.7.0 // indirect
+	golang.org/x/mod v0.9.0 // indirect
+	golang.org/x/net v0.8.0 // indirect
 	golang.org/x/oauth2 v0.1.0 // indirect
 	golang.org/x/sync v0.1.0 // indirect
-	golang.org/x/sys v0.5.0 // indirect
-	golang.org/x/term v0.5.0 // indirect
-	golang.org/x/text v0.7.0 // indirect
-	golang.org/x/tools v0.2.0 // indirect
+	golang.org/x/sys v0.6.0 // indirect
+	golang.org/x/term v0.6.0 // indirect
+	golang.org/x/text v0.8.0 // indirect
+	golang.org/x/tools v0.6.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	gonum.org/v1/gonum v0.7.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect

go.sum

@@ -34,7 +34,7 @@ type PackageDetector = func(packages []types.Package, image *source.Source, lm *
 var detectors []PackageDetector
 
 func init() {
-	detectors = []PackageDetector{nodePackageDetector()}
+	detectors = []PackageDetector{}
 }
 
 func AdditionalPackages(packages []types.Package, image *source.Source, lm *types.LayerMapping) []types.Package {

sbom/detect/detect.go

@@ -0,0 +1,41 @@
+/*
+ * Copyright © 2022 Docker, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package detect
+
+import (
+	"regexp"
+
+	"github.com/docker/index-cli-plugin/types"
+)
+
+func pythonPackageDetector() PackageDetector {
+	expr := regexp.MustCompile("libpython(.*)\\.so\\.")
+	pkg := types.Package{
+		Type:        "github",
+		Namespace:   "python",
+		Name:        "python",
+		Author:      "Python Project",
+		Description: "Python runtime",
+		Licenses:    []string{"PSF"},
+		Url:         "https://python.org",
+	}
+	filter := func(purl string) bool {
+		pkg, _ := types.ToPackageUrl(purl)
+		return pkg.Name == "python"
+	}
+	return stringsNodeDetector("python", "PYTHON_VERSION", expr, pkg, filter)
+}

sbom/detect/python.go

@@ -81,7 +81,7 @@ func IndexPath(path string, name string, cli command.Cli) (*types.Sbom, error) {
 func IndexImage(image string, options IndexOptions) (*types.Sbom, error) {
 	if strings.HasPrefix(image, "sha256:") {
 		configFilePath := options.Cli.ConfigFile().Filename
-		sbomFilePath := filepath.Join(filepath.Dir(configFilePath), "sbom", "sha256", image[7:], "sbom.json")
+		sbomFilePath := filepath.Join(filepath.Dir(configFilePath), "scout", "sbom", "sha256", image[7:], "sbom.json")
 		if sbom := cachedSbom(sbomFilePath); sbom != nil {
 			return sbom, nil
 		}
@@ -95,7 +95,7 @@ func IndexImage(image string, options IndexOptions) (*types.Sbom, error) {
 
 func indexImage(cache *registry.ImageCache, cli command.Cli) (*types.Sbom, error) {
 	configFilePath := cli.ConfigFile().Filename
-	sbomFilePath := filepath.Join(filepath.Dir(configFilePath), "sbom", "sha256", cache.Id[7:], "sbom.json")
+	sbomFilePath := filepath.Join(filepath.Dir(configFilePath), "scout", "sbom", "sha256", cache.Id[7:], "sbom.json")
 	if sbom := cachedSbom(sbomFilePath); sbom != nil {
 		return sbom, nil
 	}
@@ -136,7 +136,7 @@ func indexImage(cache *registry.ImageCache, cli command.Cli) (*types.Sbom, error
 		return nil, errors.Wrapf(err, "failed to normalize packages: %s", cache.Name)
 	}
 
-	packages := types.MergePackages(syftResult, trivyResult)
+	packages := types.FilterGenericPackages(types.MergePackages(syftResult, trivyResult))
 
 	s.Stop()
 	skill.Log.Infof(`Indexed %d packages`, len(packages))

sbom/index.go

@@ -22,7 +22,6 @@ import (
 	"strings"
 
 	"github.com/anchore/packageurl-go"
-
 	"github.com/atomist-skills/go-skill"
 )
 
@@ -203,6 +202,39 @@ func MergePackages(results ...IndexResult) []Package {
 	return packages
 }
 
+func FilterGenericPackages(packages []Package) []Package {
+	pkgs := make([]Package, 0)
+	genericPkgs := make([]Package, 0)
+	for _, pkg := range packages {
+		if pkg.Type != "generic" {
+			pkgs = append(pkgs, pkg)
+		} else {
+			genericPkgs = append(genericPkgs, pkg)
+		}
+	}
+	for _, pkg := range genericPkgs {
+		found := false
+		for _, loc := range pkg.Locations {
+			for _, p := range pkgs {
+				if containsLocation(p.Locations, loc.Path) || containsLocation(p.Files, loc.Path) {
+					found = true
+				}
+			}
+		}
+		for _, loc := range pkg.Files {
+			for _, p := range pkgs {
+				if containsLocation(p.Locations, loc.Path) || containsLocation(p.Files, loc.Path) {
+					found = true
+				}
+			}
+		}
+		if !found {
+			pkgs = append(pkgs, pkg)
+		}
+	}
+	return pkgs
+}
+
 func containsPackage(packages *[]Package, pkg Package) (int, bool) {
 	for i, p := range *packages {
 		if p.Purl == pkg.Purl {