Cleaning up FS/tar utils

Author: cftorres

utils/fs_utils.go

@@ -2,11 +2,11 @@ package utils
 
 import (
 	"bytes"
-	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/golang/glog"
 )
@@ -22,18 +22,33 @@ func GetDirectorySize(path string) (int64, error) {
 	return size, err
 }
 
-func GetDirectory(dirpath string) (Directory, error) {
-	dirfile, err := ioutil.ReadFile(dirpath)
-	if err != nil {
-		return Directory{}, err
-	}
+// GetDirectoryContents converts the directory starting at the provided path into a Directory struct.
+func GetDirectory(path string, deep bool) (Directory, error) {
+	var directory Directory
+	directory.Root = path
+	var err error
+	if deep {
+		walkFn := func(currPath string, info os.FileInfo, err error) error {
+			newContent := strings.TrimPrefix(currPath, directory.Root)
+			if newContent != "" {
+				directory.Content = append(directory.Content, newContent)
+			}
+			return nil
+		}
 
-	var dir Directory
-	err = json.Unmarshal(dirfile, &dir)
-	if err != nil {
-		return Directory{}, err
+		err = filepath.Walk(path, walkFn)
+	} else {
+		contents, err := ioutil.ReadDir(path)
+		if err != nil {
+			return directory, err
+		}
+
+		for _, file := range contents {
+			fileName := "/" + file.Name()
+			directory.Content = append(directory.Content, fileName)
+		}
 	}
-	return dir, nil
+	return directory, err
 }
 
 // Checks for content differences between files of the same name from different directories
@@ -83,12 +98,19 @@ type DirDiff struct {
 	Mods   []string
 }
 
-func compareDirEntries(d1, d2 Directory) DirDiff {
+func DiffDirectory(d1, d2 Directory) (DirDiff, bool) {
 	adds := GetAddedEntries(d1, d2)
 	dels := GetDeletedEntries(d1, d2)
 	mods := GetModifiedEntries(d1, d2)
+	
+	var same bool
+	if len(adds) == 0 && len(dels) == 0 && len(mods) == 0 {
+		same = true
+	} else {
+		same = false
+	}
 
-	return DirDiff{d1.Root, d2.Root, adds, dels, mods}
+	return DirDiff{d1.Root, d2.Root, adds, dels, mods}, same
 }
 
 func checkSameFile(f1name, f2name string) (bool, error) {
@@ -121,7 +143,3 @@ func checkSameFile(f1name, f2name string) (bool, error) {
 	}
 	return true, nil
 }
-
-func DiffDirectory(d1, d2 Directory) DirDiff {
-	return compareDirEntries(d1, d2)
-}

utils/fs_utils_test.go

@@ -68,25 +68,42 @@ func TestGetModifiedEntries(t *testing.T) {
 }
 
 func TestGetDirectory(t *testing.T) {
-	type dirtestpair struct {
-		input            string
-		expected_success bool
+	tests := []struct {
+		descrip  string
+		path     string
+		expected Directory
+		deep     bool
+	}{
+		{
+			descrip: "deep", 
+			path: "testTars/la-croix3-full",
+			expected: Directory{
+				Root: "testTars/la-croix3-full", 
+				Content: []string{"/lime.txt","/nest","/nest/f1.txt","/nested-dir","/nested-dir/f2.txt","/passionfruit.txt","/peach-pear.txt"},
+			},
+			deep: true,
+		},
+		{
+			descrip: "shallow", 
+			path: "testTars/la-croix3-full",
+			expected: Directory{
+				Root: "testTars/la-croix3-full", 
+				Content: []string{"/lime.txt","/nest","/nested-dir","/passionfruit.txt","/peach-pear.txt"},
+			},
+			deep: false,
+		},
 	}
+	for _, testCase := range tests {
+		dir, err := GetDirectory(testCase.path, testCase.deep)
+		if err != nil {
+			t.Errorf("Error converting directory to Directory struct")
+		}
 
-	var dirtests = []dirtestpair{
-		{"test_files/dir.json", true},
-		{"test_files/dir_bad.json", false},
-		{"nonexistentpath", false},
-		{"", false},
-	}
+		actualDir := dir
+		expectedDir := testCase.expected
 
-	for _, test := range dirtests {
-		_, err := GetDirectory(test.input)
-		if err != nil && test.expected_success {
-			t.Errorf("Got unexpected error: %s", err)
-		}
-		if err == nil && !test.expected_success {
-			t.Errorf("Expected error but got none")
+		if !reflect.DeepEqual(actualDir, expectedDir) {
+			t.Errorf("%s test was incorrect\nExpected: %s\nGot: %s", testCase.descrip, expectedDir, actualDir)
 		}
 	}
 }

utils/tar_utils.go

@@ -2,9 +2,7 @@ package utils
 
 import (
 	"archive/tar"
-	"encoding/json"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -77,9 +75,9 @@ func unpackTar(tr *tar.Reader, path string) error {
 
 // UnTar takes in a path to a tar file and writes the untarred version to the provided target.
 // Only untars one level, does not untar nested tars.
-func UnTar(filename string, path string) error {
-	if _, ok := os.Stat(path); ok != nil {
-		os.MkdirAll(path, 0777)
+func UnTar(filename string, target string) error {
+	if _, ok := os.Stat(target); ok != nil {
+		os.MkdirAll(target, 0777)
 	}
 
 	file, err := os.Open(filename)
@@ -88,7 +86,7 @@ func UnTar(filename string, path string) error {
 	}
 	defer file.Close()
 	tr := tar.NewReader(file)
-	err = unpackTar(tr, path)
+	err = unpackTar(tr, target)
 	if err != nil {
 		glog.Error(err)
 		return err
@@ -124,55 +122,6 @@ func ExtractTar(path string) error {
 	return filepath.Walk(path, untarWalkFn)
 }
 
-func TarToDir(tarPath string, deep bool) (string, string, error) {
-	err := ExtractTar(tarPath)
-	if err != nil {
-		return "", "", err
-	}
-	path := strings.TrimSuffix(tarPath, filepath.Ext(tarPath))
-	jsonPath := path + ".json"
-	err = DirToJSON(path, jsonPath, deep)
-	if err != nil {
-		return "", "", err
-	}
-	return jsonPath, path, nil
-}
-
-// DirToJSON records the directory structure starting at the provided path as in a json file.
-func DirToJSON(path string, target string, deep bool) error {
-	var directory Directory
-	directory.Root = path
-
-	if deep {
-		tarJSONWalkFn := func(currPath string, info os.FileInfo, err error) error {
-			newContent := strings.TrimPrefix(currPath, directory.Root)
-			if newContent != "" {
-				directory.Content = append(directory.Content, newContent)
-			}
-			return nil
-		}
-
-		filepath.Walk(path, tarJSONWalkFn)
-	} else {
-		contents, err := ioutil.ReadDir(path)
-		if err != nil {
-			return err
-		}
-
-		for _, file := range contents {
-			fileName := "/" + file.Name()
-			directory.Content = append(directory.Content, fileName)
-		}
-	}
-
-	data, err := json.Marshal(directory)
-	if err != nil {
-		return err
-	}
-
-	return ioutil.WriteFile(target, data, 0777)
-}
-
 func CheckTar(image string) bool {
 	if strings.TrimSuffix(image, ".tar") == image {
 		return false

utils/tar_utils_test.go

@@ -1,15 +1,12 @@
 package utils
 
 import (
-	"bytes"
-	"encoding/json"
+	"fmt"
 	"errors"
 	"io"
 	"io/ioutil"
 	"log"
 	"os"
-	"path/filepath"
-	"reflect"
 	"testing"
 )
 
@@ -54,22 +51,39 @@ func TestUnTar(t *testing.T) {
 			expected: "testTars/la-croix-update-actual",
 			starter:  "testTars/la-croix-starter",
 		},
+		{
+			descrip:  "Dir updated",
+			tarPath:  "testTars/la-croix-dir-update.tar",
+			target:   "testTars/la-croix-dir-update",
+			expected: "testTars/la-croix-dir-update-actual",
+			starter:  "testTars/la-croix-starter",
+		},
 	}
 	for _, test := range testCases {
+		remove := true
 		if test.starter != "" {
 			CopyDir(test.starter, test.target)
 		}
 		err := UnTar(test.tarPath, test.target)
 		if err != nil && !test.err {
 			t.Errorf(test.descrip, "Got unexpected error: %s", err)
+			remove = false
 		}
 		if err == nil && test.err {
 			t.Errorf(test.descrip, "Expected error but got none: %s", err)
+			remove = false
+		}
+		if !dirEquals(test.expected, test.target) {
+			d1, _ := GetDirectory(test.expected, true)
+			fmt.Println(d1.Content)
+			d2, _ := GetDirectory(test.target, true)
+			fmt.Println(d2.Content)
+			t.Error(test.descrip, ": Directory created not correct structure.")
+			remove = false
 		}
-		if !dirEquals(test.expected, test.target) || !dirEquals(test.target, test.expected) {
-			t.Errorf(test.descrip, "Directory created not correct structure.")
+		if remove {
+			os.RemoveAll(test.target)
 		}
-		os.RemoveAll(test.target)
 	}
 }
 
@@ -164,7 +178,7 @@ func TestIsTar(t *testing.T) {
 	}
 }
 
-func TestExtractTar(t *testing.T) {
+/*func TestExtractTar(t *testing.T) {
 	tarPath := "testTars/la-croix3.tar"
 	target := "testTars/la-croix3"
 	expected := "testTars/la-croix3-full"
@@ -177,70 +191,11 @@ func TestExtractTar(t *testing.T) {
 	}
 	os.RemoveAll(target)
 
-}
+}*/
 
 func dirEquals(actual string, path string) bool {
-	files1, _ := ioutil.ReadDir(actual)
-
-	for _, file := range files1 {
-		newActualPath := filepath.Join(actual, file.Name())
-		newExpectedPath := filepath.Join(path, file.Name())
-		fstat, ok := os.Stat(newExpectedPath)
-		if ok != nil {
-			return false
-		}
-
-		if file.IsDir() && !dirEquals(newActualPath, newExpectedPath) {
-			return false
-		}
-
-		if fstat.Name() != file.Name() {
-			return false
-		}
-		if fstat.Size() != file.Size() {
-			return false
-		}
-		if filepath.Ext(file.Name()) == ".tar" {
-			continue
-		}
-
-		content1, _ := ioutil.ReadFile(newActualPath)
-		content2, _ := ioutil.ReadFile(newExpectedPath)
-
-		if 0 != bytes.Compare(content1, content2) {
-			return false
-		}
-	}
-	return true
-}
-
-func TestDirToJSON(t *testing.T) {
-	tests := []struct {
-		path     string
-		target   string
-		expected string
-		deep     bool
-	}{
-		{"testTars/la-croix3-full", "testTars/la-croix3-full.json", "testTars/la-croix3-actual.json", true},
-		{"testTars/la-croix3-full", "testTars/la-croix3-temp.json", "testTars/la-croix3-shallow.json", false},
-	}
-	for _, testCase := range tests {
-		err := DirToJSON(testCase.path, testCase.target, testCase.deep)
-		if err != nil {
-			t.Errorf("Error converting structure to JSON")
-		}
-
-		var actualJSON Directory
-		var expectedJSON Directory
-		content1, _ := ioutil.ReadFile(testCase.target)
-		content2, _ := ioutil.ReadFile(testCase.expected)
-
-		json.Unmarshal(content1, &actualJSON)
-		json.Unmarshal(content2, &expectedJSON)
-
-		if !reflect.DeepEqual(actualJSON, expectedJSON) {
-			t.Errorf("JSON was incorrect")
-		}
-		os.Remove(testCase.target)
-	}
+	d1, _ := GetDirectory(actual, true)
+	d2, _ := GetDirectory(path, true)
+	_, same := DiffDirectory(d1, d2)
+	return same
 }