Skip to content
This repository was archived by the owner on Jul 18, 2025. It is now read-only.

Commit 4205c9f

Browse files
cdupuiscdupuis
committed
Refactor to use less memory
1 parent 7af84f9 commit 4205c9f

File tree

6 files changed

+122
-104
lines changed

6 files changed

+122
-104
lines changed

‎registry/save.go‎

Lines changed: 67 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -19,22 +19,23 @@ package registry
1919
import (
2020
"context"
2121
"fmt"
22+
"io"
2223
"os"
2324
"path/filepath"
2425
"strings"
2526

27+
stereoscopeimage "github.com/anchore/stereoscope/pkg/image"
28+
"github.com/anchore/syft/syft/source"
2629
"github.com/atomist-skills/go-skill"
2730
"github.com/docker/cli/cli/command"
2831
"github.com/docker/index-cli-plugin/internal"
29-
"github.com/dustin/go-humanize"
3032
"github.com/google/go-containerregistry/pkg/authn"
3133
"github.com/google/go-containerregistry/pkg/name"
3234
v1 "github.com/google/go-containerregistry/pkg/v1"
3335
"github.com/google/go-containerregistry/pkg/v1/daemon"
3436
"github.com/google/go-containerregistry/pkg/v1/empty"
3537
"github.com/google/go-containerregistry/pkg/v1/layout"
3638
"github.com/google/go-containerregistry/pkg/v1/remote"
37-
"github.com/google/go-containerregistry/pkg/v1/tarball"
3839
"github.com/google/uuid"
3940
"github.com/pkg/errors"
4041
)
@@ -70,11 +71,13 @@ type ImageCache struct {
7071
Tags []string
7172

7273
Image *v1.Image
74+
Source *source.Source
7375
ImagePath string
7476
Ref *name.Reference
7577

76-
copy bool
77-
cli command.Cli
78+
copy bool
79+
cli command.Cli
80+
sourceCleanup func()
7881
}
7982

8083
func (c *ImageCache) StoreImage() error {
@@ -96,53 +99,78 @@ func (c *ImageCache) StoreImage() error {
9699
if err = p.AppendImage(*c.Image); err != nil {
97100
return err
98101
}
102+
103+
input := source.Input{
104+
Scheme: source.ImageScheme,
105+
ImageSource: stereoscopeimage.OciDirectorySource,
106+
Location: c.ImagePath,
107+
}
108+
src, cleanup, err := source.New(input, nil, nil)
109+
if err != nil {
110+
return errors.Wrap(err, "failed to create new source")
111+
}
112+
c.Source = src
113+
c.sourceCleanup = cleanup
114+
99115
spinner.Stop()
100116
skill.Log.Infof("Copied image")
101117
return nil
118+
102119
} else if format == "tar" {
103-
u := make(chan v1.Update, 0)
104-
errchan := make(chan error)
105-
go func() {
106-
if err := tarball.WriteToFile(c.ImagePath, *c.Ref, *c.Image, tarball.WithProgress(u)); err != nil {
107-
errchan <- errors.Wrapf(err, "failed to write tmp image archive")
120+
spinner := internal.StartSpinner("info", "Copying image", c.cli.Out().IsTerminal())
121+
defer spinner.Stop()
122+
tempTarFile, err := os.Create(c.ImagePath)
123+
if err != nil {
124+
return errors.Wrap(err, "unable to create temp file for image")
125+
}
126+
defer func() {
127+
err := tempTarFile.Close()
128+
if err != nil {
129+
skill.Log.Errorf("unable to close temp file (%s): %w", tempTarFile.Name(), err)
108130
}
109-
errchan <- nil
110131
}()
111132

112-
var update v1.Update
113-
var err error
114-
var pp int64
115-
spinner := internal.StartSpinner("info", "Copying image", c.cli.Out().IsTerminal())
116-
defer spinner.Stop()
117-
for {
118-
select {
119-
case update = <-u:
120-
if update.Total > 0 {
121-
p := 100 * update.Complete / update.Total
122-
if pp != p {
123-
spinner.WithFields(internal.Fields{
124-
"event": "progress",
125-
"total": update.Total,
126-
"complete": update.Complete,
127-
}).Update(fmt.Sprintf("Copying image %d%% %s/%s", p, humanize.Bytes(uint64(update.Complete)), humanize.Bytes(uint64(update.Total))))
128-
pp = p
129-
}
130-
}
131-
case err = <-errchan:
132-
if err != nil {
133-
return err
134-
} else {
135-
spinner.Stop()
136-
skill.Log.Infof("Copied image")
137-
return nil
138-
}
133+
readCloser, err := c.cli.Client().ImageSave(context.Background(), []string{c.Id})
134+
if err != nil {
135+
return errors.Wrap(err, "unable to save image tar")
136+
}
137+
defer func() {
138+
err := readCloser.Close()
139+
if err != nil {
140+
skill.Log.Errorf("unable to close temp file (%s): %w", tempTarFile.Name(), err)
139141
}
142+
}()
143+
144+
nBytes, err := io.Copy(tempTarFile, readCloser)
145+
if err != nil {
146+
return fmt.Errorf("unable to save image to tar: %w", err)
147+
}
148+
if nBytes == 0 {
149+
return errors.New("cannot provide an empty image")
150+
}
151+
152+
input := source.Input{
153+
Scheme: source.ImageScheme,
154+
ImageSource: stereoscopeimage.DockerTarballSource,
155+
Location: c.ImagePath,
140156
}
157+
src, cleanup, err := source.New(input, nil, nil)
158+
if err != nil {
159+
return errors.Wrap(err, "failed to create new source")
160+
}
161+
c.Source = src
162+
c.sourceCleanup = cleanup
163+
164+
spinner.Stop()
165+
skill.Log.Infof("Copied image")
141166
}
142167
return nil
143168
}
144169

145170
func (c *ImageCache) Cleanup() {
171+
if c.sourceCleanup != nil {
172+
c.sourceCleanup()
173+
}
146174
if !c.copy {
147175
return
148176
}
@@ -169,7 +197,7 @@ func SaveImage(image string, cli command.Cli) (*ImageCache, error) {
169197
}
170198
tarPath := filepath.Join(path, "sha256", digest[7:])
171199
tarFileName := filepath.Join(tarPath, uuid.NewString())
172-
if os.Getenv("ATOMIST_CACHE_FORMAT") == "tar" {
200+
if os.Getenv("ATOMIST_CACHE_FORMAT") != "oci" {
173201
tarFileName += ".tar"
174202
}
175203

@@ -204,6 +232,7 @@ func SaveImage(image string, cli command.Cli) (*ImageCache, error) {
204232
name = strings.Split(t, ":")[0]
205233
tags = append(tags, strings.Split(t, ":")[1])
206234
}
235+
207236
return &ImageCache{
208237
Id: im.ID,
209238
Digest: digest,

‎sbom/detect/detect.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,15 +28,15 @@ import (
2828
"github.com/docker/index-cli-plugin/types"
2929
)
3030

31-
type PackageDetector = func(packages []types.Package, image source.Source, lm types.LayerMapping) []types.Package
31+
type PackageDetector = func(packages []types.Package, image *source.Source, lm *types.LayerMapping) []types.Package
3232

3333
var detectors []PackageDetector
3434

3535
func init() {
3636
detectors = []PackageDetector{nodePackageDetector()}
3737
}
3838

39-
func AdditionalPackages(packages []types.Package, image source.Source, lm types.LayerMapping) []types.Package {
39+
func AdditionalPackages(packages []types.Package, image *source.Source, lm *types.LayerMapping) []types.Package {
4040
additionalPackages := make([]types.Package, 0)
4141
for _, d := range detectors {
4242
additionalPackages = append(additionalPackages, d(packages, image, lm)...)
@@ -45,7 +45,7 @@ func AdditionalPackages(packages []types.Package, image source.Source, lm types.
4545
}
4646

4747
func stringsNodeDetector(executable string, versionEnvVar string, expr *regexp.Regexp, pkg types.Package, filterFunc func(purl string) bool) PackageDetector {
48-
return func(packages []types.Package, image source.Source, lm types.LayerMapping) []types.Package {
48+
return func(packages []types.Package, image *source.Source, lm *types.LayerMapping) []types.Package {
4949
// Already found via package manager
5050
for _, p := range packages {
5151
if filterFunc(p.Purl) {

‎sbom/index.go‎

Lines changed: 36 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -87,26 +87,28 @@ func indexImage(cache *registry.ImageCache, cli command.Cli) (*types.Sbom, error
8787
err := cache.StoreImage()
8888
defer cache.Cleanup()
8989
if err != nil {
90-
return nil, errors.Wrapf(err, "failed to copy image")
90+
return nil, errors.Wrap(err, "failed to copy image")
9191
}
9292

93-
lm := createLayerMapping(*cache.Image)
94-
93+
lm, err := createLayerMapping(cache)
94+
if err != nil {
95+
return nil, errors.Wrap(err, "failed to index image")
96+
}
9597
s := internal.StartSpinner("info", "Indexing", cli.Out().IsTerminal())
9698
defer s.Stop()
9799
trivyResultChan := make(chan types.IndexResult)
98100
syftResultChan := make(chan types.IndexResult)
99-
go trivySbom(cache.ImagePath, lm, trivyResultChan)
100-
go syftSbom(cache.ImagePath, lm, syftResultChan)
101+
go trivySbom(cache, lm, trivyResultChan)
102+
go syftSbom(cache, lm, syftResultChan)
101103

102104
trivyResult := <-trivyResultChan
103105
syftResult := <-syftResultChan
104106

105107
if trivyResult.Error != nil {
106-
return nil, errors.Wrapf(trivyResult.Error, "failed to index image")
108+
return nil, errors.Wrap(trivyResult.Error, "failed to index image")
107109
}
108110
if syftResult.Error != nil {
109-
return nil, errors.Wrapf(syftResult.Error, "failed to index image")
111+
return nil, errors.Wrap(syftResult.Error, "failed to index image")
110112
}
111113

112114
trivyResult.Packages, err = types.NormalizePackages(trivyResult.Packages)
@@ -120,10 +122,14 @@ func indexImage(cache *registry.ImageCache, cli command.Cli) (*types.Sbom, error
120122
s.Stop()
121123
skill.Log.Infof(`Indexed %d packages`, len(packages))
122124

123-
manifest, _ := (*cache.Image).RawManifest()
124-
config, _ := (*cache.Image).RawConfigFile()
125-
c, _ := (*cache.Image).ConfigFile()
126-
m, _ := (*cache.Image).Manifest()
125+
rawManifest := cache.Source.Image.Metadata.RawManifest
126+
rawConfig := cache.Source.Image.Metadata.RawConfig
127+
128+
var manifest v1.Manifest
129+
err = json.Unmarshal(rawManifest, &manifest)
130+
if err != nil {
131+
return nil, errors.Wrap(err, "failed to unmarshal manifest")
132+
}
127133

128134
sbom := types.Sbom{
129135
Artifacts: packages,
@@ -132,17 +138,17 @@ func indexImage(cache *registry.ImageCache, cli command.Cli) (*types.Sbom, error
132138
Image: types.ImageSource{
133139
Name: cache.Name,
134140
Digest: cache.Digest,
135-
Manifest: m,
136-
Config: c,
137-
RawManifest: base64.StdEncoding.EncodeToString(manifest),
138-
RawConfig: base64.StdEncoding.EncodeToString(config),
141+
Manifest: &manifest,
142+
Config: &cache.Source.Image.Metadata.Config,
143+
RawManifest: base64.StdEncoding.EncodeToString(rawManifest),
144+
RawConfig: base64.StdEncoding.EncodeToString(rawConfig),
139145
Distro: syftResult.Distro,
140146
Platform: types.Platform{
141-
Os: c.OS,
142-
Architecture: c.Architecture,
143-
Variant: c.Variant,
147+
Os: cache.Source.Image.Metadata.Config.OS,
148+
Architecture: cache.Source.Image.Metadata.Config.Architecture,
149+
Variant: cache.Source.Image.Metadata.Config.Variant,
144150
},
145-
Size: m.Config.Size,
151+
Size: cache.Source.Image.Metadata.Size,
146152
},
147153
},
148154
Descriptor: types.Descriptor{
@@ -160,11 +166,11 @@ func indexImage(cache *registry.ImageCache, cli command.Cli) (*types.Sbom, error
160166
if err == nil {
161167
err = os.MkdirAll(filepath.Dir(sbomFilePath), os.ModePerm)
162168
if err != nil {
163-
return nil, errors.Wrapf(err, "failed create to sbom folder")
169+
return nil, errors.Wrap(err, "failed create to sbom folder")
164170
}
165171
err = os.WriteFile(sbomFilePath, js, 0644)
166172
if err != nil {
167-
return nil, errors.Wrapf(err, "failed to write sbom")
173+
return nil, errors.Wrap(err, "failed to write sbom")
168174
}
169175
}
170176

@@ -191,30 +197,30 @@ func cachedSbom(sbomFilePath string) *types.Sbom {
191197
return nil
192198
}
193199

194-
func createLayerMapping(img v1.Image) types.LayerMapping {
200+
func createLayerMapping(cache *registry.ImageCache) (*types.LayerMapping, error) {
201+
skill.Log.Debugf("Creating layer mapping")
195202
lm := types.LayerMapping{
196203
ByDiffId: make(map[string]string, 0),
197204
ByDigest: make(map[string]string, 0),
198205
DiffIdByOrdinal: make(map[int]string, 0),
199206
DigestByOrdinal: make(map[int]string, 0),
200207
OrdinalByDiffId: make(map[string]int, 0),
201208
}
202-
config, _ := img.ConfigFile()
203-
diffIds := config.RootFS.DiffIDs
204-
manifest, _ := img.Manifest()
205-
layers := manifest.Layers
209+
210+
diffIds := cache.Source.Image.Metadata.Config.RootFS.DiffIDs
211+
layers := cache.Source.Metadata.ImageMetadata.Layers
206212

207213
for i := range layers {
208214
layer := layers[i]
209215
diffId := diffIds[i]
210216

211-
lm.ByDiffId[diffId.String()] = layer.Digest.String()
212-
lm.ByDigest[layer.Digest.String()] = diffId.String()
217+
lm.ByDiffId[diffId.String()] = layer.Digest
218+
lm.ByDigest[layer.Digest] = diffId.String()
213219
lm.OrdinalByDiffId[diffId.String()] = i
214220
lm.DiffIdByOrdinal[i] = diffId.String()
215-
lm.DigestByOrdinal[i] = layer.Digest.String()
221+
lm.DigestByOrdinal[i] = layer.Digest
216222
}
217223

218224
skill.Log.Debugf("Created layer mapping")
219-
return lm
225+
return &lm, nil
220226
}

0 commit comments

Comments
 (0)