Add filesystem caching

Author: nkubala

Gopkg.lock

@@ -21,6 +21,7 @@ go_library(
         "//vendor/github.com/google/go-containerregistry/pkg/v1/daemon:go_default_library",
         "//vendor/github.com/google/go-containerregistry/pkg/v1/remote:go_default_library",
         "//vendor/github.com/google/go-containerregistry/pkg/v1/tarball:go_default_library",
+        "//vendor/github.com/mitchellh/go-homedir:go_default_library",
         "//vendor/github.com/sirupsen/logrus:go_default_library",
         "//vendor/github.com/spf13/cobra:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",

cmd/BUILD.bazel

@@ -64,7 +64,7 @@ func analyzeImage(imageName string, analyzerArgs []string) error {
 		return err
 	}
 
-	if !save {
+	if noCache && !save {
 		defer pkgutil.CleanupImage(image)
 	}
 	if err != nil {
@@ -79,11 +79,11 @@ func analyzeImage(imageName string, analyzerArgs []string) error {
 		return fmt.Errorf("Error performing image analysis: %s", err)
 	}
 
-	output.PrintToStdErr("Retrieving analyses\n")
+	logrus.Info("retrieving analyses")
 	outputResults(analyses)
 
-	if save {
-		logrus.Infof("Image was saved at %s", image.FSPath)
+	if noCache && save {
+		logrus.Infof("image was saved at %s", image.FSPath)
 	}
 
 	return nil

cmd/analyze.go

@@ -78,7 +78,7 @@ func diffImages(image1Arg, image2Arg string, diffArgs []string) error {
 	var wg sync.WaitGroup
 	wg.Add(2)
 
-	output.PrintToStdErr("Starting diff on images %s and %s, using differs: %s\n", image1Arg, image2Arg, diffArgs)
+	logrus.Infof("starting diff on images %s and %s, using differs: %s\n", image1Arg, image2Arg, diffArgs)
 
 	imageMap := map[string]*pkgutil.Image{
 		image1Arg: {},
@@ -97,12 +97,12 @@ func diffImages(image1Arg, image2Arg string, diffArgs []string) error {
 	}
 	wg.Wait()
 
-	if !save {
+	if noCache && !save {
 		defer pkgutil.CleanupImage(*imageMap[image1Arg])
 		defer pkgutil.CleanupImage(*imageMap[image2Arg])
 	}
 
-	output.PrintToStdErr("Computing diffs\n")
+	logrus.Info("computing diffs")
 	req := differs.DiffRequest{
 		Image1:    *imageMap[image1Arg],
 		Image2:    *imageMap[image2Arg],
@@ -114,14 +114,14 @@ func diffImages(image1Arg, image2Arg string, diffArgs []string) error {
 	outputResults(diffs)
 
 	if filename != "" {
-		output.PrintToStdErr("Computing filename diffs\n")
+		logrus.Info("computing filename diffs")
 		err := diffFile(imageMap[image1Arg], imageMap[image2Arg])
 		if err != nil {
 			return err
 		}
 	}
 
-	if save {
+	if noCache && save {
 		logrus.Infof("Images were saved at %s and %s", imageMap[image1Arg].FSPath,
 			imageMap[image2Arg].FSPath)
 	}

cmd/diff.go

@@ -22,8 +22,10 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
+	"path/filepath"
 	"sort"
 	"strings"
+	"time"
 
 	"github.com/google/go-containerregistry/pkg/authn"
 	"github.com/google/go-containerregistry/pkg/name"
@@ -35,14 +37,17 @@ import (
 	pkgutil "github.com/GoogleContainerTools/container-diff/pkg/util"
 	"github.com/GoogleContainerTools/container-diff/util"
 	"github.com/google/go-containerregistry/pkg/v1"
+	homedir "github.com/mitchellh/go-homedir"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 )
 
 var json bool
+
 var save bool
 var types diffTypes
+var noCache bool
 
 var LogLevel string
 var format string
@@ -125,14 +130,17 @@ func checkIfValidAnalyzer(_ []string) error {
 }
 
 func getImageForName(imageName string) (pkgutil.Image, error) {
-	logrus.Infof("getting image for name %s", imageName)
+	logrus.Infof("retrieving image: %s", imageName)
 	var img v1.Image
 	var err error
 	if pkgutil.IsTar(imageName) {
+		start := time.Now()
 		img, err = tarball.ImageFromPath(imageName, nil)
 		if err != nil {
 			return pkgutil.Image{}, err
 		}
+		elapsed := time.Now().Sub(start)
+		logrus.Infof("retrieving image from tar took %f seconds", elapsed.Seconds())
 	}
 
 	if strings.HasPrefix(imageName, DaemonPrefix) {
@@ -144,10 +152,16 @@ func getImageForName(imageName string) (pkgutil.Image, error) {
 			return pkgutil.Image{}, err
 		}
 
-		img, err = daemon.Image(ref, &daemon.ReadOptions{})
+		start := time.Now()
+		// TODO(nkubala): specify gzip.NoCompression here when functional options are supported
+		img, err = daemon.Image(ref, &daemon.ReadOptions{
+			Buffer: true,
+		})
 		if err != nil {
 			return pkgutil.Image{}, err
 		}
+		elapsed := time.Now().Sub(start)
+		logrus.Infof("retrieving image from daemon took %f seconds", elapsed.Seconds())
 	} else {
 		// either has remote prefix or has no prefix, in which case we force remote
 		imageName = strings.Replace(imageName, RemotePrefix, "", -1)
@@ -159,22 +173,27 @@ func getImageForName(imageName string) (pkgutil.Image, error) {
 		if err != nil {
 			return pkgutil.Image{}, err
 		}
+		start := time.Now()
 		img, err = remote.Image(ref, auth, http.DefaultTransport)
 		if err != nil {
 			return pkgutil.Image{}, err
 		}
+		elapsed := time.Now().Sub(start)
+		logrus.Infof("retrieving remote image took %f seconds", elapsed.Seconds())
 	}
-	// TODO(nkubala): implement caching
 
 	// create tempdir and extract fs into it
 	var layers []pkgutil.Layer
 	if includeLayers() {
+		start := time.Now()
 		imgLayers, err := img.Layers()
 		if err != nil {
 			return pkgutil.Image{}, err
 		}
 		for _, layer := range imgLayers {
-			path, err := ioutil.TempDir("", strings.Replace(imageName, "/", "", -1))
+			layerStart := time.Now()
+			digest, err := layer.Digest()
+			path, err := getExtractPathForName(digest.String())
 			if err != nil {
 				return pkgutil.Image{
 					Layers: layers,
@@ -188,12 +207,15 @@ func getImageForName(imageName string) (pkgutil.Image, error) {
 			layers = append(layers, pkgutil.Layer{
 				FSPath: path,
 			})
+			elapsed := time.Now().Sub(layerStart)
+			logrus.Infof("time elapsed retrieving layer: %fs", elapsed.Seconds())
 		}
+		elapsed := time.Now().Sub(start)
+		logrus.Infof("time elapsed retrieving image layers: %fs", elapsed.Seconds())
 	}
-	path, err := ioutil.TempDir("", strings.Replace(imageName, "/", "", -1))
-	if err != nil {
-		return pkgutil.Image{}, err
-	}
+
+	path, err := getExtractPathForImage(imageName, img)
+	// extract fs into provided dir
 	if err := pkgutil.GetFileSystemForImage(img, path, nil); err != nil {
 		return pkgutil.Image{
 			FSPath: path,
@@ -208,6 +230,44 @@ func getImageForName(imageName string) (pkgutil.Image, error) {
 	}, nil
 }
 
+func getExtractPathForImage(imageName string, image v1.Image) (string, error) {
+	start := time.Now()
+	digest, err := image.Digest()
+	if err != nil {
+		return "", err
+	}
+	elapsed := time.Now().Sub(start)
+	logrus.Infof("time elapsed retrieving image digest: %fs", elapsed.Seconds())
+	return getExtractPathForName(pkgutil.RemoveTag(imageName) + "@" + digest.String())
+}
+
+func getExtractPathForName(name string) (string, error) {
+	var path string
+	var err error
+	if !noCache {
+		path, err = cacheDir(name)
+		if err != nil {
+			return "", err
+		}
+		// if cachedir doesn't exist, create it
+		if _, err := os.Stat(path); err != nil && os.IsNotExist(err) {
+			err = os.MkdirAll(path, 0700)
+			if err != nil {
+				return "", err
+			}
+			logrus.Infof("caching filesystem at %s", path)
+		}
+	} else {
+		// otherwise, create tempdir
+		logrus.Infof("skipping caching")
+		path, err = ioutil.TempDir("", strings.Replace(name, "/", "", -1))
+		if err != nil {
+			return "", err
+		}
+	}
+	return path, nil
+}
+
 func includeLayers() bool {
 	for _, t := range types {
 		if t == "layer" {
@@ -217,6 +277,16 @@ func includeLayers() bool {
 	return false
 }
 
+func cacheDir(imageName string) (string, error) {
+	dir, err := homedir.Dir()
+	if err != nil {
+		return "", err
+	}
+	rootDir := filepath.Join(dir, ".container-diff", "cache")
+	imageName = strings.Replace(imageName, string(os.PathSeparator), "", -1)
+	return filepath.Join(rootDir, filepath.Clean(imageName)), nil
+}
+
 func init() {
 	RootCmd.PersistentFlags().StringVarP(&LogLevel, "verbosity", "v", "warning", "This flag controls the verbosity of container-diff.")
 	RootCmd.PersistentFlags().StringVarP(&format, "format", "", "", "Format to output diff in.")
@@ -254,4 +324,5 @@ func addSharedFlags(cmd *cobra.Command) {
 	cmd.Flags().VarP(&types, "type", "t", "This flag sets the list of analyzer types to use. Set it repeatedly to use multiple analyzers.")
 	cmd.Flags().BoolVarP(&save, "save", "s", false, "Set this flag to save rather than remove the final image filesystems on exit.")
 	cmd.Flags().BoolVarP(&util.SortSize, "order", "o", false, "Set this flag to sort any file/package results by descending size. Otherwise, they will be sorted by name.")
+	cmd.Flags().BoolVarP(&noCache, "no-cache", "n", false, "Set this to force retrieval of image filesystem on each run.")
 }

cmd/root.go

@@ -18,6 +18,7 @@ package util
 
 import (
 	"bytes"
+	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -191,3 +192,18 @@ func HasFilepathPrefix(path, prefix string) bool {
 	}
 	return true
 }
+
+// given a path to a directory, check if it has any contents
+func DirIsEmpty(path string) (bool, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return false, err
+	}
+	defer f.Close()
+
+	_, err = f.Readdir(1)
+	if err == io.EOF {
+		return true, nil
+	}
+	return false, err
+}

pkg/util/fs_utils.go

@@ -33,6 +33,8 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+const tagRegexStr = ".*:([^/]+$)"
+
 type Layer struct {
 	FSPath string
 }
@@ -83,7 +85,16 @@ func GetFileSystemForLayer(layer v1.Layer, root string, whitelist []string) erro
 }
 
 // unpack image filesystem to local disk
+// if provided directory is not empty, do nothing
 func GetFileSystemForImage(image v1.Image, root string, whitelist []string) error {
+	empty, err := DirIsEmpty(root)
+	if err != nil {
+		return err
+	}
+	if !empty {
+		logrus.Infof("using cached filesystem in %s", root)
+		return nil
+	}
 	if err := unpackTar(tar.NewReader(mutate.Extract(image)), root, whitelist); err != nil {
 		return err
 	}
@@ -134,6 +145,17 @@ func copyToFile(outfile string, r io.Reader) error {
 
 // checks to see if an image string contains a tag.
 func HasTag(image string) bool {
-	tagRegex := regexp.MustCompile(".*:[^/]+$")
+	tagRegex := regexp.MustCompile(tagRegexStr)
 	return tagRegex.MatchString(image)
 }
+
+// returns a raw image name with the tag removed
+func RemoveTag(image string) string {
+	if !HasTag(image) {
+		return image
+	}
+	tagRegex := regexp.MustCompile(tagRegexStr)
+	parts := tagRegex.FindStringSubmatch(image)
+	tag := parts[len(parts)-1]
+	return image[0 : len(image)-len(tag)-1]
+}