Add a "MutableSource" type that allows for basic image modifications. (#180)

Author: dlorenc

cmd/root.go

@@ -172,7 +172,7 @@ func getPrepperForImage(image string) (pkgutil.Prepper, error) {
 			return nil, err
 		}
 
-		src, err = cache.NewFileCache(cacheDir, ref, src)
+		src, err = cache.NewFileCache(cacheDir, ref)
 		if err != nil {
 			return nil, err
 		}

pkg/cache/BUILD.bazel

@@ -6,8 +6,8 @@ go_library(
     importpath = "github.com/GoogleCloudPlatform/container-diff/pkg/cache",
     visibility = ["//visibility:public"],
     deps = [
+        "//pkg/image:go_default_library",
         "//vendor/github.com/containers/image/types:go_default_library",
-        "//vendor/github.com/opencontainers/go-digest:go_default_library",
         "//vendor/github.com/sirupsen/logrus:go_default_library",
     ],
 )

pkg/cache/file_cache.go

@@ -17,32 +17,34 @@ limitations under the License.
 package cache
 
 import (
-	"context"
 	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 
+	"github.com/GoogleCloudPlatform/container-diff/pkg/image"
 	"github.com/containers/image/types"
-	digest "github.com/opencontainers/go-digest"
 	"github.com/sirupsen/logrus"
 )
 
 type FileCache struct {
 	RootDir string
-	Ref     types.ImageReference
-	src     types.ImageSource
+	*image.ProxySource
 }
 
-func NewFileCache(dir string, ref types.ImageReference, src types.ImageSource) (*FileCache, error) {
+func NewFileCache(dir string, ref types.ImageReference) (*FileCache, error) {
 	if err := os.MkdirAll(dir, 0700); err != nil {
 		return nil, err
 	}
 
+	ps, err := image.NewProxySource(ref)
+	if err != nil {
+		return nil, err
+	}
+
 	return &FileCache{
-		RootDir: dir,
-		Ref:     ref,
-		src:     src,
+		RootDir:     dir,
+		ProxySource: ps,
 	}, nil
 }
 
@@ -80,24 +82,6 @@ func (c *FileCache) Invalidate(layer types.BlobInfo) error {
 	return os.RemoveAll(filepath.Join(c.RootDir, layerId))
 }
 
-// Implement types.ImageSource
-
-func (c *FileCache) Reference() types.ImageReference {
-	return c.Ref
-}
-
-func (c *FileCache) Close() error {
-	return nil
-}
-
-func (c *FileCache) GetManifest(d *digest.Digest) ([]byte, string, error) {
-	return c.src.GetManifest(d)
-}
-
-func (c *FileCache) GetTargetManifest(digest digest.Digest) ([]byte, string, error) {
-	return c.GetTargetManifest(digest)
-}
-
 // GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown).
 // The Digest field in BlobInfo is guaranteed to be provided; Size may be -1.
 func (c *FileCache) GetBlob(bi types.BlobInfo) (io.ReadCloser, int64, error) {
@@ -106,7 +90,7 @@ func (c *FileCache) GetBlob(bi types.BlobInfo) (io.ReadCloser, int64, error) {
 		return r, bi.Size, err
 	}
 	// Add to the cache then return
-	r, size, err := c.src.GetBlob(bi)
+	r, size, err := c.ProxySource.GetBlob(bi)
 	if err != nil {
 		return nil, 0, err
 	}
@@ -116,12 +100,3 @@ func (c *FileCache) GetBlob(bi types.BlobInfo) (io.ReadCloser, int64, error) {
 	}
 	return r, size, err
 }
-
-// GetSignatures returns the image's signatures.  It may use a remote (= slow) service.
-func (c *FileCache) GetSignatures(ctx context.Context, d *digest.Digest) ([][]byte, error) {
-	return c.src.GetSignatures(ctx, d)
-}
-
-func (c *FileCache) LayerInfosForCopy() []types.BlobInfo {
-	return nil
-}

pkg/image/BUILD.bazel

@@ -0,0 +1,28 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "mutable_source.go",
+        "proxy_types.go",
+    ],
+    importpath = "github.com/GoogleCloudPlatform/container-diff/pkg/image",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//vendor/github.com/containers/image/manifest:go_default_library",
+        "//vendor/github.com/containers/image/types:go_default_library",
+        "//vendor/github.com/opencontainers/go-digest:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["mutable_source_test.go"],
+    embed = [":go_default_library"],
+    importpath = "github.com/GoogleCloudPlatform/container-diff/pkg/image",
+    deps = [
+        "//vendor/github.com/containers/image/manifest:go_default_library",
+        "//vendor/github.com/containers/image/types:go_default_library",
+        "//vendor/github.com/opencontainers/go-digest:go_default_library",
+    ],
+)

pkg/image/mutable_source.go

@@ -0,0 +1,165 @@
+/*
+Copyright 2017 Google, Inc. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package image
+
+import (
+	"bytes"
+	"compress/gzip"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"time"
+
+	"github.com/containers/image/manifest"
+	"github.com/containers/image/types"
+	digest "github.com/opencontainers/go-digest"
+)
+
+type MutableSource struct {
+	ProxySource
+	mfst        *manifest.Schema2
+	cfg         *manifest.Schema2Image
+	extraBlobs  map[string][]byte
+	extraLayers []digest.Digest
+}
+
+func NewMutableSource(r types.ImageReference) (*MutableSource, error) {
+	src, err := r.NewImageSource(nil)
+	if err != nil {
+		return nil, err
+	}
+	img, err := r.NewImage(nil)
+	if err != nil {
+		return nil, err
+	}
+
+	ms := &MutableSource{
+		ProxySource: ProxySource{
+			Ref:         r,
+			ImageSource: src,
+			img:         img,
+		},
+		extraBlobs: make(map[string][]byte),
+	}
+	if err := ms.populateManifestAndConfig(); err != nil {
+		return nil, err
+	}
+	return ms, nil
+}
+
+// GetManifest marshals the stored manifest to the byte format.
+func (m *MutableSource) GetManifest(_ *digest.Digest) ([]byte, string, error) {
+	s, err := json.Marshal(m.mfst)
+	if err := m.saveConfig(); err != nil {
+		return nil, "", err
+	}
+	return s, manifest.DockerV2Schema2MediaType, err
+}
+
+// populateManifestAndConfig parses the raw manifest and configs, storing them on the struct.
+func (m *MutableSource) populateManifestAndConfig() error {
+	mfstBytes, _, err := m.GetManifest(nil)
+	if err != nil {
+		return err
+	}
+
+	m.mfst, err = manifest.Schema2FromManifest(mfstBytes)
+	if err != nil {
+		return err
+	}
+
+	bi := types.BlobInfo{Digest: m.mfst.ConfigDescriptor.Digest}
+	r, _, err := m.GetBlob(bi)
+	if err != nil {
+		return err
+	}
+
+	cfgBytes, err := ioutil.ReadAll(r)
+	if err != nil {
+		return err
+	}
+
+	return json.Unmarshal(cfgBytes, &m.cfg)
+}
+
+// GetBlob first checks the stored "extra" blobs, then proxies the call to the original source.
+func (m *MutableSource) GetBlob(bi types.BlobInfo) (io.ReadCloser, int64, error) {
+	if b, ok := m.extraBlobs[bi.Digest.String()]; ok {
+		return ioutil.NopCloser(bytes.NewReader(b)), int64(len(b)), nil
+	}
+	return m.ImageSource.GetBlob(bi)
+}
+
+func gzipBytes(b []byte) ([]byte, error) {
+	buf := bytes.NewBuffer([]byte{})
+	w := gzip.NewWriter(buf)
+	_, err := w.Write(b)
+	w.Close()
+	if err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
+
+// appendLayer appends an uncompressed blob to the image, preserving the invariants required across the config and manifest.
+func (m *MutableSource) appendLayer(content []byte) error {
+	compressedBlob, err := gzipBytes(content)
+	if err != nil {
+		return err
+	}
+
+	dgst := digest.FromBytes(compressedBlob)
+
+	// Add the layer to the manifest.
+	descriptor := manifest.Schema2Descriptor{
+		MediaType: manifest.DockerV2Schema2LayerMediaType,
+		Size:      int64(len(content)),
+		Digest:    dgst,
+	}
+	m.mfst.LayersDescriptors = append(m.mfst.LayersDescriptors, descriptor)
+
+	m.extraBlobs[dgst.String()] = compressedBlob
+	m.extraLayers = append(m.extraLayers, dgst)
+
+	// Also add it to the config.
+	diffID := digest.FromBytes(content)
+	m.cfg.RootFS.DiffIDs = append(m.cfg.RootFS.DiffIDs, diffID)
+	history := manifest.Schema2History{
+		Created: time.Now(),
+		Author:  "container-diff",
+	}
+	m.cfg.History = append(m.cfg.History, history)
+
+	return nil
+}
+
+// saveConfig marshals the stored image config, and updates the references to it in the manifest.
+func (m *MutableSource) saveConfig() error {
+	cfgBlob, err := json.Marshal(m.cfg)
+	if err != nil {
+		return err
+	}
+
+	cfgDigest := digest.FromBytes(cfgBlob)
+	m.extraBlobs[cfgDigest.String()] = cfgBlob
+	m.mfst.ConfigDescriptor = manifest.Schema2Descriptor{
+		MediaType: manifest.DockerV2Schema2ConfigMediaType,
+		Size:      int64(len(cfgBlob)),
+		Digest:    cfgDigest,
+	}
+	return nil
+}