Skip to content
This repository was archived by the owner on Mar 27, 2024. It is now read-only.

Commit ea27f9c

Browse files
dlorencdlorenc
authored
Refactor the caching and image unpacking. (#174)
This will make image unpacking from the registry easier to re-use outside of this library.
1 parent eb9635a commit ea27f9c

File tree

90 files changed

+851
-828
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

90 files changed

+851
-828
lines changed

‎BUILD

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,8 @@
1-
load("@io_bazel_rules_go//go:def.bzl", "gazelle", "go_binary", "go_library", "go_prefix")
1+
load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_prefix")
2+
load("@bazel_gazelle//:def.bzl", "gazelle")
23

34
gazelle(
45
name = "gazelle",
5-
build_tags = [
6-
"container_image_ostree_stub",
7-
"containers_image_openpgp",
8-
],
9-
external = "vendored",
10-
mode = "fix",
116
prefix = "github.com/GoogleCloudPlatform/container-diff",
127
)
138

‎WORKSPACE

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,13 @@ http_archive(
33
url = "https://github.com/bazelbuild/rules_go/releases/download/0.9.0/rules_go-0.9.0.tar.gz",
44
sha256 = "4d8d6244320dd751590f9100cf39fd7a4b75cd901e1f3ffdfd6f048328883695",
55
)
6-
6+
http_archive(
7+
name = "bazel_gazelle",
8+
url = "https://github.com/bazelbuild/bazel-gazelle/releases/download/0.8/bazel-gazelle-0.8.tar.gz",
9+
sha256 = "e3dadf036c769d1f40603b86ae1f0f90d11837116022d9b06e4cd88cae786676",
10+
)
711
load("@io_bazel_rules_go//go:def.bzl", "go_rules_dependencies", "go_register_toolchains")
812
go_rules_dependencies()
913
go_register_toolchains()
14+
load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
15+
gazelle_dependencies()

‎cmd/BUILD.bazel

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,13 @@ go_library(
1616
"//pkg/cache:go_default_library",
1717
"//pkg/util:go_default_library",
1818
"//util:go_default_library",
19-
"//vendor/github.com/docker/docker/client:go_default_library",
20-
"//vendor/github.com/mitchellh/go-homedir:go_default_library",
21-
"//vendor/github.com/sirupsen/logrus:go_default_library",
22-
"//vendor/github.com/spf13/cobra:go_default_library",
23-
"//vendor/github.com/spf13/pflag:go_default_library",
2419
"//version:go_default_library",
20+
"@com_github_containers_image//docker:go_default_library",
21+
"@com_github_docker_docker//client:go_default_library",
22+
"@com_github_mitchellh_go_homedir//:go_default_library",
23+
"@com_github_sirupsen_logrus//:go_default_library",
24+
"@com_github_spf13_cobra//:go_default_library",
25+
"@com_github_spf13_pflag//:go_default_library",
2526
],
2627
)
2728

‎cmd/root.go

Lines changed: 33 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import (
2828
"github.com/GoogleCloudPlatform/container-diff/pkg/cache"
2929
pkgutil "github.com/GoogleCloudPlatform/container-diff/pkg/util"
3030
"github.com/GoogleCloudPlatform/container-diff/util"
31+
"github.com/containers/image/docker"
3132
"github.com/docker/docker/client"
3233
homedir "github.com/mitchellh/go-homedir"
3334
"github.com/sirupsen/logrus"
@@ -126,26 +127,24 @@ func getPrepperForImage(image string) (pkgutil.Prepper, error) {
126127
return nil, err
127128
}
128129

129-
cacheDir, err := cacheDir()
130-
if err != nil {
131-
return nil, err
132-
}
133-
var fsCache cache.Cache
134-
if !noCache {
135-
fsCache, err = cache.NewFileCache(cacheDir)
136-
if err != nil {
137-
return nil, err
138-
}
139-
}
140-
141130
if pkgutil.IsTar(image) {
142131
return pkgutil.TarPrepper{
143132
Source: image,
144133
Client: cli,
145-
Cache: fsCache,
146134
}, nil
147135

148-
} else if strings.HasPrefix(image, DaemonPrefix) {
136+
}
137+
138+
ref, err := docker.ParseReference("//" + image)
139+
if err != nil {
140+
return nil, err
141+
}
142+
src, err := ref.NewImageSource(nil, nil)
143+
if err != nil {
144+
return nil, err
145+
}
146+
147+
if strings.HasPrefix(image, DaemonPrefix) {
149148

150149
// remove the DaemonPrefix
151150
image := strings.Replace(image, DaemonPrefix, "", -1)
@@ -155,17 +154,31 @@ func getPrepperForImage(image string) (pkgutil.Prepper, error) {
155154
}
156155

157156
return pkgutil.DaemonPrepper{
158-
Source: image,
159-
Client: cli,
160-
Cache: fsCache,
157+
Source: image,
158+
Client: cli,
159+
ImageSource: src,
161160
}, nil
162161
}
163162
// either has remote prefix or has no prefix, in which case we force remote
163+
164+
if !noCache {
165+
cacheDir, err := cacheDir()
166+
if err != nil {
167+
return nil, err
168+
}
169+
170+
src, err = cache.NewFileCache(cacheDir, ref, src)
171+
if err != nil {
172+
return nil, err
173+
}
174+
}
175+
164176
return pkgutil.CloudPrepper{
165-
Source: strings.Replace(image, RemotePrefix, "", -1),
166-
Client: cli,
167-
Cache: fsCache,
177+
Source: strings.Replace(image, RemotePrefix, "", -1),
178+
Client: cli,
179+
ImageSource: src,
168180
}, nil
181+
169182
}
170183

171184
func cacheDir() (string, error) {

‎cmd/util/output/BUILD.bazel

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,5 +5,5 @@ go_library(
55
srcs = ["output.go"],
66
importpath = "github.com/GoogleCloudPlatform/container-diff/cmd/util/output",
77
visibility = ["//visibility:public"],
8-
deps = ["//vendor/github.com/spf13/cobra:go_default_library"],
8+
deps = ["@com_github_spf13_cobra//:go_default_library"],
99
)

‎differs/BUILD.bazel

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,10 @@ go_library(
1717
deps = [
1818
"//pkg/util:go_default_library",
1919
"//util:go_default_library",
20-
"//vendor/github.com/containers/image/docker/tarfile:go_default_library",
21-
"//vendor/github.com/fsouza/go-dockerclient:go_default_library",
22-
"//vendor/github.com/nightlyone/lockfile:go_default_library",
23-
"//vendor/github.com/sirupsen/logrus:go_default_library",
20+
"@com_github_containers_image//docker/tarfile:go_default_library",
21+
"@com_github_fsouza_go_dockerclient//:go_default_library",
22+
"@com_github_nightlyone_lockfile//:go_default_library",
23+
"@com_github_sirupsen_logrus//:go_default_library",
2424
],
2525
)
2626

‎pkg/cache/BUILD.bazel

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,12 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library")
22

33
go_library(
44
name = "go_default_library",
5-
srcs = [
6-
"cache.go",
7-
"file_cache.go",
8-
],
5+
srcs = ["file_cache.go"],
96
importpath = "github.com/GoogleCloudPlatform/container-diff/pkg/cache",
107
visibility = ["//visibility:public"],
11-
deps = ["//vendor/github.com/sirupsen/logrus:go_default_library"],
8+
deps = [
9+
"@com_github_containers_image//types:go_default_library",
10+
"@com_github_opencontainers_go_digest//:go_default_library",
11+
"@com_github_sirupsen_logrus//:go_default_library",
12+
],
1213
)

‎pkg/cache/cache.go

Lines changed: 0 additions & 26 deletions
This file was deleted.

‎pkg/cache/file_cache.go

Lines changed: 63 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,30 +17,42 @@ limitations under the License.
1717
package cache
1818

1919
import (
20+
"context"
2021
"io"
2122
"os"
2223
"path/filepath"
2324

25+
"github.com/containers/image/types"
26+
digest "github.com/opencontainers/go-digest"
2427
"github.com/sirupsen/logrus"
2528
)
2629

2730
type FileCache struct {
2831
RootDir string
32+
Ref types.ImageReference
33+
src types.ImageSource
2934
}
3035

31-
func NewFileCache(dir string) (*FileCache, error) {
36+
func NewFileCache(dir string, ref types.ImageReference, src types.ImageSource) (*FileCache, error) {
3237
if err := os.MkdirAll(dir, 0700); err != nil {
3338
return nil, err
3439
}
35-
return &FileCache{RootDir: dir}, nil
40+
41+
return &FileCache{
42+
RootDir: dir,
43+
Ref: ref,
44+
src: src,
45+
}, nil
3646
}
3747

38-
func (c *FileCache) HasLayer(layerId string) bool {
48+
func (c *FileCache) HasLayer(layer types.BlobInfo) bool {
49+
layerId := layer.Digest.String()
3950
_, err := os.Stat(filepath.Join(c.RootDir, layerId))
4051
return !os.IsNotExist(err)
4152
}
4253

43-
func (c *FileCache) SetLayer(layerId string, r io.Reader) (io.ReadCloser, error) {
54+
func (c *FileCache) SetLayer(layer types.BlobInfo, r io.Reader) (io.ReadCloser, error) {
55+
layerId := layer.Digest.String()
4456
fullpath := filepath.Join(c.RootDir, layerId)
4557
entry, err := os.Create(fullpath)
4658
if err != nil {
@@ -49,14 +61,58 @@ func (c *FileCache) SetLayer(layerId string, r io.Reader) (io.ReadCloser, error)
4961
if _, err := io.Copy(entry, r); err != nil {
5062
return nil, err
5163
}
52-
return c.GetLayer(layerId)
64+
return c.GetLayer(layer)
5365
}
5466

55-
func (c *FileCache) GetLayer(layerId string) (io.ReadCloser, error) {
67+
func (c *FileCache) GetLayer(layer types.BlobInfo) (io.ReadCloser, error) {
68+
layerId := layer.Digest.String()
5669
logrus.Infof("retrieving layer %s from cache", layerId)
5770
return os.Open(filepath.Join(c.RootDir, layerId))
5871
}
5972

60-
func (c *FileCache) Invalidate(layerId string) error {
73+
func (c *FileCache) Invalidate(layer types.BlobInfo) error {
74+
layerId := layer.Digest.String()
6175
return os.RemoveAll(filepath.Join(c.RootDir, layerId))
6276
}
77+
78+
// Implement types.ImageSource
79+
80+
func (c *FileCache) Reference() types.ImageReference {
81+
return c.Ref
82+
}
83+
84+
func (c *FileCache) Close() error {
85+
return nil
86+
}
87+
88+
func (c *FileCache) GetManifest() ([]byte, string, error) {
89+
return c.src.GetManifest()
90+
}
91+
92+
func (c *FileCache) GetTargetManifest(digest digest.Digest) ([]byte, string, error) {
93+
return c.GetTargetManifest(digest)
94+
}
95+
96+
// GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown).
97+
// The Digest field in BlobInfo is guaranteed to be provided; Size may be -1.
98+
func (c *FileCache) GetBlob(bi types.BlobInfo) (io.ReadCloser, int64, error) {
99+
if c.HasLayer(bi) {
100+
r, err := c.GetLayer(bi)
101+
return r, bi.Size, err
102+
}
103+
// Add to the cache then return
104+
r, size, err := c.src.GetBlob(bi)
105+
if err != nil {
106+
return nil, 0, err
107+
}
108+
r, err = c.SetLayer(bi, r)
109+
if err != nil {
110+
return nil, 0, c.Invalidate(bi)
111+
}
112+
return r, size, err
113+
}
114+
115+
// GetSignatures returns the image's signatures. It may use a remote (= slow) service.
116+
func (c *FileCache) GetSignatures(ctx context.Context) ([][]byte, error) {
117+
return c.src.GetSignatures(ctx)
118+
}

‎pkg/util/BUILD.bazel

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,14 @@ go_library(
1616
visibility = ["//visibility:public"],
1717
deps = [
1818
"//cmd/util/output:go_default_library",
19-
"//pkg/cache:go_default_library",
20-
"//vendor/github.com/containers/image/docker:go_default_library",
21-
"//vendor/github.com/containers/image/docker/daemon:go_default_library",
22-
"//vendor/github.com/containers/image/docker/tarfile:go_default_library",
23-
"//vendor/github.com/containers/image/manifest:go_default_library",
24-
"//vendor/github.com/containers/image/pkg/compression:go_default_library",
25-
"//vendor/github.com/containers/image/types:go_default_library",
26-
"//vendor/github.com/docker/docker/client:go_default_library",
27-
"//vendor/github.com/docker/docker/pkg/system:go_default_library",
28-
"//vendor/github.com/sirupsen/logrus:go_default_library",
19+
"@com_github_containers_image//docker:go_default_library",
20+
"@com_github_containers_image//docker/daemon:go_default_library",
21+
"@com_github_containers_image//docker/tarfile:go_default_library",
22+
"@com_github_containers_image//manifest:go_default_library",
23+
"@com_github_containers_image//pkg/compression:go_default_library",
24+
"@com_github_containers_image//types:go_default_library",
25+
"@com_github_docker_docker//client:go_default_library",
26+
"@com_github_docker_docker//pkg/system:go_default_library",
27+
"@com_github_sirupsen_logrus//:go_default_library",
2928
],
3029
)

0 commit comments

Comments
 (0)