Adding unpacking of tars into final FS

Author: cftorres

differs/fileDiff.go

@@ -1,11 +1,9 @@
 package differs
 
 import (
-	"os"
 	"sort"
 
 	"github.com/GoogleCloudPlatform/container-diff/utils"
-	"github.com/golang/glog"
 )
 
 type FileDiffer struct {
@@ -23,21 +21,11 @@ func diffImageFiles(image1, image2 utils.Image) (utils.DirDiff, error) {
 
 	var diff utils.DirDiff
 
-	target1 := "j1.json"
-	err := utils.DirToJSON(img1, target1, true)
+	img1Dir, err := utils.GetDirectory(img1, true)
 	if err != nil {
 		return diff, err
 	}
-	target2 := "j2.json"
-	err = utils.DirToJSON(img2, target2, true)
-	if err != nil {
-		return diff, err
-	}
-	img1Dir, err := utils.GetDirectory(target1)
-	if err != nil {
-		return diff, err
-	}
-	img2Dir, err := utils.GetDirectory(target2)
+	img2Dir, err := utils.GetDirectory(img2, true)
 	if err != nil {
 		return diff, err
 	}
@@ -52,16 +40,7 @@ func diffImageFiles(image1, image2 utils.Image) (utils.DirDiff, error) {
 		Image2: image2.Source,
 		Adds:   adds,
 		Dels:   dels,
-		Mods:	[]string{},
-	}
-
-	err = os.Remove(target1)
-	if err != nil {
-		glog.Error(err)
-	}
-	err = os.Remove(target2)
-	if err != nil {
-		glog.Error(err)
+		Mods:   []string{},
 	}
 	return diff, nil
 }

differs/pipDiff.go

@@ -75,10 +75,7 @@ func (d PipDiffer) getPackages(image utils.Image) (map[string]map[string]utils.P
 	for _, pythonPath := range pythonPaths {
 		contents, err := ioutil.ReadDir(pythonPath)
 		if err != nil {
-<<<<<<< HEAD
 			// python version folder doesn't have a site-packages folder or PYTHONPATH doesn't exist
-=======
->>>>>>> Addressing Abby's comment
 			continue
 		}
 

tests/file_diff_expected.json

@@ -8,7 +8,7 @@
       "Dels": [
         "/home/test"
       ],
-      "Mods": null
+      "Mods": []
     }
   }
 ]

utils/docker_utils.go

@@ -5,9 +5,13 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
+	"os"
 	"os/exec"
+	"path/filepath"
 	"regexp"
 	"strings"
 	"syscall"
@@ -36,6 +40,7 @@ func ValidDockerVersion() (bool, error) {
 	}
 	return false, nil
 }
+
 func getImagePullResponse(image string, response []Event) (string, error) {
 	var imageDigest string
 	for _, event := range response {
@@ -303,3 +308,59 @@ func getImageConfig(image string) (container.Config, error) {
 	}
 	return config, nil
 }
+
+func getLayersFromManifest(manifestPath string) ([]string, error) {
+	type Manifest struct {
+		Layers []string
+	}
+
+	manifestJSON, err := ioutil.ReadFile(manifestPath)
+	if err != nil {
+		errMsg := fmt.Sprintf("Could not open manifest to get layer order: %s", err)
+		return []string{}, errors.New(errMsg)
+	}
+
+	var imageManifest []Manifest
+	err = json.Unmarshal(manifestJSON, &imageManifest)
+	if err != nil {
+		errMsg := fmt.Sprintf("Could not unmarshal manifest to get layer order: %s", err)
+		return []string{}, errors.New(errMsg)
+	}
+	return imageManifest[0].Layers, nil
+}
+
+func unpackDockerSave(tarPath string, target string) error {
+	if _, ok := os.Stat(target); ok != nil {
+		os.MkdirAll(target, 0777)
+	}
+
+	tempLayerDir := target + "-temp"
+	err := UnTar(tarPath, tempLayerDir)
+	if err != nil {
+		errMsg := fmt.Sprintf("Could not unpack saved Docker image %s: %s", tarPath, err)
+		return errors.New(errMsg)
+	}
+
+	manifest := filepath.Join(tempLayerDir, "manifest.json")
+	layers, err := getLayersFromManifest(manifest)
+	if err != nil {
+		return err
+	}
+
+	for _, layer := range layers {
+		layerTar := filepath.Join(tempLayerDir, layer)
+		if _, err := os.Stat(layerTar); err != nil {
+			glog.Infof("Did not unpack layer %s because no layer.tar found", layer)
+			continue
+		}
+		err = UnTar(layerTar, target)
+		if err != nil {
+			glog.Errorf("Could not unpack layer %s: %s", layer, err)
+		}
+	}
+	err = os.RemoveAll(tempLayerDir)
+	if err != nil {
+		glog.Errorf("Error deleting temp image layer filesystem: %s", err)
+	}
+	return nil
+}

utils/fs_utils.go

@@ -68,6 +68,22 @@ func GetModifiedEntries(d1, d2 Directory) []string {
 			glog.Errorf("Error checking directory entry %s: %s\n", f, err)
 			continue
 		}
+		f2stat, err := os.Stat(f2path)
+		if err != nil {
+			glog.Errorf("Error checking directory entry %s: %s\n", f, err)
+			continue
+		}
+
+		// If the directory entry in question is a tar, verify that the two have the same size
+		if isTar(f1path) {
+			if f1stat.Size() != f2stat.Size() {
+				modified = append(modified, f)
+			}
+			continue
+		}
+
+		// If the directory entry is not a tar and not a directory, then it's a file so make sure the file contents are the same
+		// Note: We skip over directory entries because to compare directories, we compare their contents
 		if !f1stat.IsDir() {
 			same, err := checkSameFile(f1path, f2path)
 			if err != nil {
@@ -103,7 +119,7 @@ func DiffDirectory(d1, d2 Directory) (DirDiff, bool) {
 	adds := GetAddedEntries(d1, d2)
 	dels := GetDeletedEntries(d1, d2)
 	mods := GetModifiedEntries(d1, d2)
-	
+
 	var same bool
 	if len(adds) == 0 && len(dels) == 0 && len(mods) == 0 {
 		same = true

utils/fs_utils_test.go

@@ -75,20 +75,20 @@ func TestGetDirectory(t *testing.T) {
 		deep     bool
 	}{
 		{
-			descrip: "deep", 
-			path: "testTars/la-croix3-full",
+			descrip: "deep",
+			path:    "testTars/la-croix3-full",
 			expected: Directory{
-				Root: "testTars/la-croix3-full", 
-				Content: []string{"/lime.txt","/nest","/nest/f1.txt","/nested-dir","/nested-dir/f2.txt","/passionfruit.txt","/peach-pear.txt"},
+				Root:    "testTars/la-croix3-full",
+				Content: []string{"/lime.txt", "/nest", "/nest/f1.txt", "/nested-dir", "/nested-dir/f2.txt", "/passionfruit.txt", "/peach-pear.txt"},
 			},
 			deep: true,
 		},
 		{
-			descrip: "shallow", 
-			path: "testTars/la-croix3-full",
+			descrip: "shallow",
+			path:    "testTars/la-croix3-full",
 			expected: Directory{
-				Root: "testTars/la-croix3-full", 
-				Content: []string{"/lime.txt","/nest","/nested-dir","/passionfruit.txt","/peach-pear.txt"},
+				Root:    "testTars/la-croix3-full",
+				Content: []string{"/lime.txt", "/nest", "/nested-dir", "/passionfruit.txt", "/peach-pear.txt"},
 			},
 			deep: false,
 		},

utils/image_prep_utils.go

@@ -95,7 +95,7 @@ func (p ImagePrepper) GetImage() (Image, error) {
 func getImageFromTar(tarPath string) (string, error) {
 	glog.Info("Extracting image tar to obtain image file system")
 	path := strings.TrimSuffix(tarPath, filepath.Ext(tarPath))
-	err := UnTar(tarPath, path)
+	err := unpackDockerSave(tarPath, path)
 	return path, err
 }
 
@@ -201,10 +201,7 @@ func (p IDPrepper) getFileSystem() (string, error) {
 	}
 
 	defer os.Remove(tarPath)
-	glog.Info("Extracting image tar to obtain image file system")
-	path := strings.TrimSuffix(tarPath, filepath.Ext(tarPath))
-	err = ExtractTar(tarPath)
-	return path, err
+	return getImageFromTar(tarPath)
 }
 
 func (p IDPrepper) getConfig() (ConfigSchema, error) {
@@ -255,13 +252,13 @@ func (p TarPrepper) getFileSystem() (string, error) {
 }
 
 func (p TarPrepper) getConfig() (ConfigSchema, error) {
-	tmpDir := strings.TrimSuffix(p.Source, filepath.Ext(p.Source))
-	defer os.Remove(tmpDir)
-	err := UnTar(p.Source, tmpDir)
+	tempDir := strings.TrimSuffix(p.Source, filepath.Ext(p.Source)) + "-config"
+	defer os.RemoveAll(tempDir)
+	err := UnTar(p.Source, tempDir)
 	if err != nil {
 		return ConfigSchema{}, err
 	}
-	contents, err := ioutil.ReadDir(tmpDir)
+	contents, err := ioutil.ReadDir(tempDir)
 	if err != nil {
 		glog.Errorf("Could not read image tar contents: %s", err)
 		return ConfigSchema{}, errors.New("Could not obtain image config")
@@ -276,7 +273,7 @@ func (p TarPrepper) getConfig() (ConfigSchema, error) {
 				glog.Error("Multiple possible config sources detected for image at " + p.Source + ". Multiple images likely contained in tar. Choosing first one, but diff results may not be completely accurate.")
 				break
 			}
-			fileName := filepath.Join(tmpDir, item.Name())
+			fileName := filepath.Join(tempDir, item.Name())
 			file, err := ioutil.ReadFile(fileName)
 			if err != nil {
 				glog.Errorf("Could not read config file %s: %s", fileName, err)

utils/tar_utils.go

@@ -79,7 +79,6 @@ func UnTar(filename string, target string) error {
 	if _, ok := os.Stat(target); ok != nil {
 		os.MkdirAll(target, 0777)
 	}
-
 	file, err := os.Open(filename)
 	if err != nil {
 		return err
@@ -98,30 +97,6 @@ func isTar(path string) bool {
 	return filepath.Ext(path) == ".tar"
 }
 
-// ExtractTar extracts the tar and any nested tar at the given path.
-// After execution the original tar file is removed and the untarred version is in it place.
-func ExtractTar(path string) error {
-	removeTar := false
-
-	var untarWalkFn func(path string, info os.FileInfo, err error) error
-
-	untarWalkFn = func(path string, info os.FileInfo, err error) error {
-		if isTar(path) {
-			target := strings.TrimSuffix(path, filepath.Ext(path))
-			UnTar(path, target)
-			if removeTar {
-				os.Remove(path)
-			}
-			// remove nested tar files that get copied but not the original tar passed
-			removeTar = true
-			filepath.Walk(target, untarWalkFn)
-		}
-		return nil
-	}
-
-	return filepath.Walk(path, untarWalkFn)
-}
-
 func CheckTar(image string) bool {
 	if strings.TrimSuffix(image, ".tar") == image {
 		return false

utils/tar_utils_test.go

@@ -1,8 +1,8 @@
 package utils
 
 import (
-	//"fmt"
 	"errors"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"log"
@@ -74,10 +74,6 @@ func TestUnTar(t *testing.T) {
 			remove = false
 		}
 		if !dirEquals(test.expected, test.target) {
-			//d1, _ := GetDirectory(test.expected, true)
-			//fmt.Println(d1.Content)
-			//d2, _ := GetDirectory(test.target, true)
-			//fmt.Println(d2.Content)
 			t.Error(test.descrip, ": Directory created not correct structure.")
 			remove = false
 		}
@@ -178,24 +174,12 @@ func TestIsTar(t *testing.T) {
 	}
 }
 
-/*func TestExtractTar(t *testing.T) {
-	tarPath := "testTars/la-croix3.tar"
-	target := "testTars/la-croix3"
-	expected := "testTars/la-croix3-full"
-	err := ExtractTar(tarPath)
-	if err != nil {
-		t.Errorf("Got unexpected error: %s", err)
-	}
-	if !dirEquals(expected, target) || !dirEquals(target, expected) {
-		t.Errorf("Directory created not correct structure.")
-	}
-	os.RemoveAll(target)
-
-}*/
-
 func dirEquals(actual string, path string) bool {
 	d1, _ := GetDirectory(actual, true)
 	d2, _ := GetDirectory(path, true)
-	_, same := DiffDirectory(d1, d2)
+	diff, same := DiffDirectory(d1, d2)
+	if !same {
+		fmt.Printf("%v", diff)
+	}
 	return same
 }