Merge pull request #8 from docker/memory-fix

Memory fix

Author: cdupuis

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/anchore/stereoscope v0.0.0-20221006201143-d24c9d626b33
 	github.com/anchore/syft v0.62.1
 	github.com/aquasecurity/trivy v0.30.4
-	github.com/atomist-skills/go-skill v0.0.6-0.20221003172518-c3d268e1f3f1
+	github.com/atomist-skills/go-skill v0.0.6-0.20221221214636-a7de163fd901
 	github.com/briandowns/spinner v1.12.0
 	github.com/docker/cli v20.10.21+incompatible
 	github.com/docker/docker v20.10.17+incompatible

go.sum

@@ -209,6 +209,8 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:l
 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ=
 github.com/atomist-skills/go-skill v0.0.6-0.20221003172518-c3d268e1f3f1 h1:EzSOh9LLtL/3IzbPUFSp/6OF4DrgiCPxCC3x3jjD9Bs=
 github.com/atomist-skills/go-skill v0.0.6-0.20221003172518-c3d268e1f3f1/go.mod h1:DRmwrZL5kG68Mn8VDw/Xr7rDhyl+laD7NFHIrQr54yo=
+github.com/atomist-skills/go-skill v0.0.6-0.20221221214636-a7de163fd901 h1:0fqUAo4MmWXnWIDCG7JBe903M3WJ+tqQetPIkVfcXKo=
+github.com/atomist-skills/go-skill v0.0.6-0.20221221214636-a7de163fd901/go.mod h1:DRmwrZL5kG68Mn8VDw/Xr7rDhyl+laD7NFHIrQr54yo=
 github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
 github.com/aws/aws-sdk-go v1.44.46 h1:BsKENvu24eXg7CWQ2wJAjKbDFkGP+hBtxKJIR3UdcB8=
 github.com/aws/aws-sdk-go v1.44.46/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=

registry/read.go

@@ -17,6 +17,8 @@
 package registry
 
 import (
+	stereoscopeimage "github.com/anchore/stereoscope/pkg/image"
+	"github.com/anchore/syft/syft/source"
 	"github.com/atomist-skills/go-skill"
 	"github.com/google/go-containerregistry/pkg/v1/layout"
 	"github.com/pkg/errors"
@@ -31,15 +33,31 @@ func ReadImage(name string, path string) (*ImageCache, error) {
 	mani, err := index.IndexManifest()
 	hash := mani.Manifests[0].Digest
 	img, _ := index.Image(hash)
+
+	skill.Log.Debugf("Parsing image")
+	input := source.Input{
+		Scheme:      source.ImageScheme,
+		ImageSource: stereoscopeimage.OciDirectorySource,
+		Location:    path,
+	}
+	src, cleanup, err := source.New(input, nil, nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create new source")
+	}
+	skill.Log.Debugf("Parse image")
 	skill.Log.Infof("Loaded image")
+	
 	return &ImageCache{
 		Id:        hash.String(),
 		Digest:    hash.String(),
 		Tags:      []string{},
 		Name:      name,
 		Image:     &img,
+		Source:    src,
 		ImagePath: path,
 		Ref:       nil,
-		copy:      false,
+
+		copy:          false,
+		sourceCleanup: cleanup,
 	}, nil
 }

registry/save.go

@@ -19,10 +19,13 @@ package registry
 import (
 	"context"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"strings"
 
+	stereoscopeimage "github.com/anchore/stereoscope/pkg/image"
+	"github.com/anchore/syft/syft/source"
 	"github.com/atomist-skills/go-skill"
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/index-cli-plugin/internal"
@@ -70,18 +73,22 @@ type ImageCache struct {
 	Tags   []string
 
 	Image     *v1.Image
+	Source    *source.Source
 	ImagePath string
 	Ref       *name.Reference
 
-	copy bool
-	cli  command.Cli
+	remote        bool
+	copy          bool
+	cli           command.Cli
+	sourceCleanup func()
 }
 
 func (c *ImageCache) StoreImage() error {
 	if !c.copy {
 		return nil
 	}
 	skill.Log.Debugf("Copying image to %s", c.ImagePath)
+	var imageSource stereoscopeimage.Source
 
 	if format := os.Getenv("ATOMIST_CACHE_FORMAT"); format == "" || format == "oci" {
 		spinner := internal.StartSpinner("info", "Copying image", c.cli.Out().IsTerminal())
@@ -96,53 +103,113 @@ func (c *ImageCache) StoreImage() error {
 		if err = p.AppendImage(*c.Image); err != nil {
 			return err
 		}
+
+		imageSource = stereoscopeimage.OciDirectorySource
+
 		spinner.Stop()
-		skill.Log.Infof("Copied image")
-		return nil
 	} else if format == "tar" {
-		u := make(chan v1.Update, 0)
-		errchan := make(chan error)
-		go func() {
-			if err := tarball.WriteToFile(c.ImagePath, *c.Ref, *c.Image, tarball.WithProgress(u)); err != nil {
-				errchan <- errors.Wrapf(err, "failed to write tmp image archive")
-			}
-			errchan <- nil
-		}()
+		if c.remote {
+			u := make(chan v1.Update, 0)
+			errchan := make(chan error)
+			go func() {
+				if err := tarball.WriteToFile(c.ImagePath, *c.Ref, *c.Image, tarball.WithProgress(u)); err != nil {
+					errchan <- errors.Wrapf(err, "failed to write tmp image archive")
+				}
+				errchan <- nil
+			}()
 
-		var update v1.Update
-		var err error
-		var pp int64
-		spinner := internal.StartSpinner("info", "Copying image", c.cli.Out().IsTerminal())
-		defer spinner.Stop()
-		for {
-			select {
-			case update = <-u:
-				if update.Total > 0 {
-					p := 100 * update.Complete / update.Total
-					if pp != p {
-						spinner.WithFields(internal.Fields{
-							"event":    "progress",
-							"total":    update.Total,
-							"complete": update.Complete,
-						}).Update(fmt.Sprintf("Copying image %d%% %s/%s", p, humanize.Bytes(uint64(update.Complete)), humanize.Bytes(uint64(update.Total))))
-						pp = p
+			var update v1.Update
+			var err error
+			var pp int64
+			spinner := internal.StartSpinner("info", "Copying image", c.cli.Out().IsTerminal())
+			defer spinner.Stop()
+			loop := true
+			for loop {
+				select {
+				case update = <-u:
+					if update.Total > 0 {
+						p := 100 * update.Complete / update.Total
+						if pp != p {
+							spinner.WithFields(internal.Fields{
+								"event":    "progress",
+								"total":    update.Total,
+								"complete": update.Complete,
+							}).Update(fmt.Sprintf("Copying image %d%% %s/%s", p, humanize.Bytes(uint64(update.Complete)), humanize.Bytes(uint64(update.Total))))
+							pp = p
+						}
+					}
+				case err = <-errchan:
+					if err != nil {
+						return err
+					} else {
+						spinner.Stop()
+						skill.Log.Infof("Copied image")
+						loop = false
 					}
 				}
-			case err = <-errchan:
+			}
+
+		} else {
+			spinner := internal.StartSpinner("info", "Copying image", c.cli.Out().IsTerminal())
+			defer spinner.Stop()
+			tempTarFile, err := os.Create(c.ImagePath)
+			if err != nil {
+				return errors.Wrap(err, "unable to create temp file for image")
+			}
+			defer func() {
+				err := tempTarFile.Close()
+				if err != nil {
+					skill.Log.Errorf("unable to close temp file (%s): %w", tempTarFile.Name(), err)
+				}
+			}()
+
+			readCloser, err := c.cli.Client().ImageSave(context.Background(), []string{c.Id})
+			if err != nil {
+				return errors.Wrap(err, "unable to save image tar")
+			}
+			defer func() {
+				err := readCloser.Close()
 				if err != nil {
-					return err
-				} else {
-					spinner.Stop()
-					skill.Log.Infof("Copied image")
-					return nil
+					skill.Log.Errorf("unable to close temp file (%s): %w", tempTarFile.Name(), err)
 				}
+			}()
+
+			nBytes, err := io.Copy(tempTarFile, readCloser)
+			if err != nil {
+				return fmt.Errorf("unable to save image to tar: %w", err)
+			}
+			if nBytes == 0 {
+				return errors.New("cannot provide an empty image")
 			}
+			spinner.Stop()
 		}
+
+		imageSource = stereoscopeimage.DockerTarballSource
 	}
+
+	skill.Log.Debugf("Parsing image")
+	input := source.Input{
+		Scheme:      source.ImageScheme,
+		ImageSource: imageSource,
+		Location:    c.ImagePath,
+	}
+	src, cleanup, err := source.New(input, nil, nil)
+	if err != nil {
+		return errors.Wrap(err, "failed to create new image source")
+	}
+	c.Source = src
+	c.sourceCleanup = cleanup
+
+	skill.Log.Debugf("Parsed image")
+	skill.Log.Infof("Copied image")
+
 	return nil
 }
 
 func (c *ImageCache) Cleanup() {
+	if c.sourceCleanup != nil {
+		c.sourceCleanup()
+	}
 	if !c.copy {
 		return
 	}
@@ -169,7 +236,7 @@ func SaveImage(image string, cli command.Cli) (*ImageCache, error) {
 		}
 		tarPath := filepath.Join(path, "sha256", digest[7:])
 		tarFileName := filepath.Join(tarPath, uuid.NewString())
-		if os.Getenv("ATOMIST_CACHE_FORMAT") == "tar" {
+		if os.Getenv("ATOMIST_CACHE_FORMAT") != "oci" {
 			tarFileName += ".tar"
 		}
 
@@ -204,6 +271,7 @@ func SaveImage(image string, cli command.Cli) (*ImageCache, error) {
 			name = strings.Split(t, ":")[0]
 			tags = append(tags, strings.Split(t, ":")[1])
 		}
+
 		return &ImageCache{
 			Id:     im.ID,
 			Digest: digest,
@@ -214,6 +282,7 @@ func SaveImage(image string, cli command.Cli) (*ImageCache, error) {
 			Ref:       &ref,
 			ImagePath: imagePath,
 			copy:      true,
+			remote:    false,
 			cli:       cli,
 		}, nil
 	}
@@ -250,6 +319,7 @@ func SaveImage(image string, cli command.Cli) (*ImageCache, error) {
 		Ref:       &ref,
 		ImagePath: imagePath,
 		copy:      true,
+		remote:    true,
 		cli:       cli,
 	}, nil
 }

sbom/detect/detect.go

@@ -28,15 +28,15 @@ import (
 	"github.com/docker/index-cli-plugin/types"
 )
 
-type PackageDetector = func(packages []types.Package, image source.Source, lm types.LayerMapping) []types.Package
+type PackageDetector = func(packages []types.Package, image *source.Source, lm *types.LayerMapping) []types.Package
 
 var detectors []PackageDetector
 
 func init() {
 	detectors = []PackageDetector{nodePackageDetector()}
 }
 
-func AdditionalPackages(packages []types.Package, image source.Source, lm types.LayerMapping) []types.Package {
+func AdditionalPackages(packages []types.Package, image *source.Source, lm *types.LayerMapping) []types.Package {
 	additionalPackages := make([]types.Package, 0)
 	for _, d := range detectors {
 		additionalPackages = append(additionalPackages, d(packages, image, lm)...)
@@ -45,7 +45,7 @@ func AdditionalPackages(packages []types.Package, image source.Source, lm types.
 }
 
 func stringsNodeDetector(executable string, versionEnvVar string, expr *regexp.Regexp, pkg types.Package, filterFunc func(purl string) bool) PackageDetector {
-	return func(packages []types.Package, image source.Source, lm types.LayerMapping) []types.Package {
+	return func(packages []types.Package, image *source.Source, lm *types.LayerMapping) []types.Package {
 		// Already found via package manager
 		for _, p := range packages {
 			if filterFunc(p.Purl) {