Merge pull request #1807 from FabianKramm/master

fix: fix recursive symlinks

Author: FabianKramm

helper/server/downstream.go

@@ -5,6 +5,7 @@ import (
 	"compress/gzip"
 	"context"
 	"github.com/loft-sh/devspace/helper/util/pingtimeout"
+	"github.com/loft-sh/devspace/pkg/util/fsutil"
 	logpkg "github.com/loft-sh/devspace/pkg/util/log"
 	"io"
 	"io/ioutil"
@@ -526,6 +527,9 @@ func walkDir(basePath string, path string, ignoreMatcher ignoreparser.IgnorePars
 
 	for _, f := range files {
 		absolutePath := filepath.Join(path, f.Name())
+		if fsutil.IsRecursiveSymlink(f, absolutePath) {
+			continue
+		}
 
 		// Stat is necessary here, because readdir does not follow symlinks and
 		// IsDir() returns false for symlinked folders

helper/server/tar.go

@@ -3,6 +3,7 @@ package server
 import (
 	"archive/tar"
 	"compress/gzip"
+	"github.com/loft-sh/devspace/pkg/util/fsutil"
 	"io"
 	"io/ioutil"
 	"os"
@@ -213,6 +214,10 @@ func tarFolder(basePath string, fileInformation *fileInformation, writtenFiles m
 
 	if !skipContents {
 		for _, f := range files {
+			if fsutil.IsRecursiveSymlink(f, path.Join(fileInformation.Name, f.Name())) {
+				continue
+			}
+
 			if err := recursiveTar(basePath, path.Join(fileInformation.Name, f.Name()), writtenFiles, tw, skipContents); err != nil {
 				return errors.Wrap(err, "recursive tar")
 			}

helper/server/upstream.go

@@ -8,6 +8,7 @@ import (
 	"github.com/loft-sh/devspace/helper/util/crc32"
 	"github.com/loft-sh/devspace/helper/util/pingtimeout"
 	"github.com/loft-sh/devspace/helper/util/stderrlog"
+	"github.com/loft-sh/devspace/pkg/util/fsutil"
 	logpkg "github.com/loft-sh/devspace/pkg/util/log"
 	"github.com/pkg/errors"
 	"google.golang.org/grpc"
@@ -191,6 +192,9 @@ func (u *Upstream) removeRecursive(absolutePath string) error {
 	// Loop over directory contents and check if we should delete the contents
 	for _, f := range files {
 		absoluteChildPath := filepath.Join(absolutePath, f.Name())
+		if fsutil.IsRecursiveSymlink(f, absoluteChildPath) {
+			continue
+		}
 
 		// Check if ignored
 		if u.ignoreMatcher != nil && !u.ignoreMatcher.RequireFullScan() && u.ignoreMatcher.Matches(absolutePath[len(u.options.UploadPath):], f.IsDir()) {

pkg/devspace/hook/upload.go

@@ -3,6 +3,7 @@ package hook
 import (
 	"archive/tar"
 	"compress/gzip"
+	"github.com/loft-sh/devspace/pkg/util/fsutil"
 	"io"
 	"io/ioutil"
 	"os"
@@ -129,6 +130,10 @@ func recursiveTar(srcBase, srcFile, destBase, destFile string, tw *tar.Writer) e
 				}
 			}
 			for _, f := range files {
+				if fsutil.IsRecursiveSymlink(f, path.Join(fpath, f.Name())) {
+					continue
+				}
+
 				if err := recursiveTar(srcBase, path.Join(srcFile, f.Name()), destBase, path.Join(destFile, f.Name()), tw); err != nil {
 					return err
 				}

pkg/devspace/sync/downstream.go

@@ -3,9 +3,11 @@ package sync
 import (
 	"context"
 	"fmt"
+	"github.com/loft-sh/devspace/pkg/util/fsutil"
 	"io"
 	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
 	"time"
@@ -437,6 +439,10 @@ func (d *downstream) deleteSafeRecursive(relativePath string, deleteChanges []*r
 
 	// Loop over directory contents and check if we should delete the contents
 	for _, f := range files {
+		if fsutil.IsRecursiveSymlink(f, path.Join(relativePath, f.Name())) {
+			continue
+		}
+
 		childRelativePath := filepath.ToSlash(filepath.Join(relativePath, f.Name()))
 		childAbsFilepath := filepath.Join(d.sync.LocalPath, childRelativePath)
 		if shouldRemoveLocal(childAbsFilepath, d.sync.fileIndex.fileMap[childRelativePath], d.sync, force) {

pkg/devspace/sync/evaluater.go

@@ -56,11 +56,9 @@ func shouldUpload(s *Sync, fileInformation *FileInformation, log log.Logger) boo
 	}
 
 	// Exclude changes on the exclude list
-	if s.ignoreMatcher != nil {
-		if s.ignoreMatcher.Matches(fileInformation.Name, fileInformation.IsDirectory) {
-			log.Debugf("Don't upload %s because it is excluded", fileInformation.Name)
-			return false
-		}
+	if s.ignoreMatcher != nil && s.ignoreMatcher.Matches(fileInformation.Name, fileInformation.IsDirectory) {
+		log.Debugf("Don't upload %s because it is excluded", fileInformation.Name)
+		return false
 	}
 
 	// Check if we already tracked the path

pkg/devspace/sync/initial.go

@@ -2,6 +2,7 @@ package sync
 
 import (
 	"github.com/loft-sh/devspace/helper/server/ignoreparser"
+	"github.com/loft-sh/devspace/pkg/util/fsutil"
 	"io/ioutil"
 	"os"
 	"path"
@@ -356,6 +357,11 @@ func (i *initialSyncer) calculateLocalDirState(absPath string, stat os.FileInfo,
 	}
 
 	for _, f := range files {
+		if fsutil.IsRecursiveSymlink(f, filepath.Join(absPath, f.Name())) {
+			i.o.Log.Debugf("Found recursive symlink at %v", filepath.Join(absPath, f.Name()))
+			continue
+		}
+
 		err := i.CalculateLocalState(filepath.Join(absPath, f.Name()), localState, ignore)
 		if err != nil {
 			return errors.Wrap(err, f.Name())

pkg/devspace/sync/tar.go

@@ -3,6 +3,7 @@ package sync
 import (
 	"archive/tar"
 	"compress/gzip"
+	"github.com/loft-sh/devspace/pkg/util/fsutil"
 	"io"
 	"io/ioutil"
 	"os"
@@ -283,8 +284,8 @@ func (a *Archiver) AddToArchive(relativePath string) error {
 }
 
 func (a *Archiver) tarFolder(target *FileInformation, targetStat os.FileInfo) error {
-	filepath := path.Join(a.basePath, target.Name)
-	files, err := ioutil.ReadDir(filepath)
+	filePath := path.Join(a.basePath, target.Name)
+	files, err := ioutil.ReadDir(filePath)
 	if err != nil {
 		// config.Logf("[Upstream] Couldn't read dir %s: %s\n", filepath, err.Error())
 		return nil
@@ -294,7 +295,7 @@ func (a *Archiver) tarFolder(target *FileInformation, targetStat os.FileInfo) er
 		// check if not excluded
 		if a.ignoreMatcher == nil || !a.ignoreMatcher.RequireFullScan() || !a.ignoreMatcher.Matches(target.Name, true) {
 			// Case empty directory
-			hdr, _ := tar.FileInfoHeader(targetStat, filepath)
+			hdr, _ := tar.FileInfoHeader(targetStat, filePath)
 			hdr.Uid = 0
 			hdr.Gid = 0
 			hdr.Mode = fillGo18FileTypeBits(int64(chmodTarEntry(os.FileMode(hdr.Mode))), targetStat)
@@ -308,6 +309,10 @@ func (a *Archiver) tarFolder(target *FileInformation, targetStat os.FileInfo) er
 	}
 
 	for _, f := range files {
+		if fsutil.IsRecursiveSymlink(f, path.Join(filePath, f.Name())) {
+			continue
+		}
+
 		if err := a.AddToArchive(path.Join(target.Name, f.Name())); err != nil {
 			return errors.Wrap(err, "recursive tar "+f.Name())
 		}
@@ -325,7 +330,7 @@ func (a *Archiver) tarFile(target *FileInformation, targetStat os.FileInfo) erro
 		}
 
 		targetStat, err = os.Stat(filepath)
-		if err != nil {
+		if err != nil || targetStat.IsDir() {
 			// We ignore open file and just treat it as okay
 			return nil
 		}

pkg/devspace/sync/upstream.go

@@ -6,7 +6,9 @@ import (
 	"compress/gzip"
 	"context"
 	"fmt"
+	"github.com/loft-sh/devspace/pkg/util/fsutil"
 	"io"
+	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
@@ -412,101 +414,138 @@ func (u *upstream) getFileInformationFromEvent(events []notify.EventInfo) ([]*Fi
 				continue
 			}
 
-			relativePath := getRelativeFromFullPath(fullPath, u.sync.LocalPath)
-
 			// Determine what kind of change we got (Create or Remove)
-			newChange, err := u.evaluateChange(relativePath, fullPath)
+			relativePath := getRelativeFromFullPath(fullPath, u.sync.LocalPath)
+			newChanges, err := u.evaluateChange(relativePath, fullPath)
 			if err != nil {
 				return nil, errors.Wrap(err, "evaluate change")
 			}
 
-			if newChange != nil {
-				changes = append(changes, newChange)
-			}
+			changes = append(changes, newChanges...)
 		}
 	}
 
 	return changes, nil
 }
 
-func (u *upstream) evaluateChange(relativePath, fullpath string) (*FileInformation, error) {
-	stat, err := os.Stat(fullpath)
+func (u *upstream) evaluateChange(relativePath, fullPath string) ([]*FileInformation, error) {
+	stat, err := os.Stat(fullPath)
+	if err != nil {
+		// Remove symlinks
+		u.RemoveSymlinks(fullPath)
 
-	// File / Folder exist -> Create File or Folder
-	// if File / Folder does not exist, we create a new remove change
-	if err == nil {
-		// Exclude changes on the upload exclude list
-		if u.sync.uploadIgnoreMatcher != nil {
-			if u.sync.uploadIgnoreMatcher.Matches(relativePath, stat.IsDir()) {
-				// Add to file map and prevent download if local file is newer than the remote one
-				if u.sync.fileIndex.fileMap[relativePath] != nil && u.sync.fileIndex.fileMap[relativePath].Mtime < stat.ModTime().Unix() {
-					// Add it to the fileMap
-					u.sync.fileIndex.fileMap[relativePath] = &FileInformation{
-						Name:        relativePath,
-						Mtime:       stat.ModTime().Unix(),
-						Mode:        stat.Mode(),
-						Size:        stat.Size(),
-						IsDirectory: stat.IsDir(),
-					}
-				}
+		// Check if we should remove path remote
+		if shouldRemoveRemote(relativePath, u.sync) {
+			// New Remove Task
+			return []*FileInformation{
+				{
+					Name: relativePath,
+				},
+			}, nil
+		}
+
+		return nil, nil
+	}
 
-				return nil, nil
+	// Exclude changes on the upload exclude list
+	if u.sync.uploadIgnoreMatcher != nil && u.sync.uploadIgnoreMatcher.Matches(relativePath, stat.IsDir()) {
+		// Add to file map and prevent download if local file is newer than the remote one
+		if u.sync.fileIndex.fileMap[relativePath] != nil && u.sync.fileIndex.fileMap[relativePath].Mtime < stat.ModTime().Unix() {
+			// Add it to the fileMap
+			u.sync.fileIndex.fileMap[relativePath] = &FileInformation{
+				Name:        relativePath,
+				Mtime:       stat.ModTime().Unix(),
+				Mode:        stat.Mode(),
+				Size:        stat.Size(),
+				IsDirectory: stat.IsDir(),
 			}
 		}
 
-		// Check if symbolic link
-		lstat, err := os.Lstat(fullpath)
-		if err == nil && lstat.Mode()&os.ModeSymlink != 0 {
-			_, symlinkExists := u.sync.upstream.symlinks[fullpath]
+		return nil, nil
+	}
+
+	// File / Folder exist -> Create File or Folder
+	// if File / Folder does not exist, we create a new remove change
+	// Check if symbolic link
+	lstat, err := os.Lstat(fullPath)
+	if err == nil && lstat.Mode()&os.ModeSymlink != 0 {
+		_, symlinkExists := u.sync.upstream.symlinks[fullPath]
+
+		// Add symlink to map
+		stat, err = u.sync.upstream.AddSymlink(relativePath, fullPath)
+		if err != nil {
+			return nil, errors.Wrap(err, "add symlink")
+		}
+		if stat == nil {
+			return nil, nil
+		}
 
-			// Add symlink to map
-			stat, err = u.sync.upstream.AddSymlink(relativePath, fullpath)
+		// Only crawl if symlink wasn't there before and it is a directory
+		if !symlinkExists && stat.IsDir() {
+			// Crawl all linked files & folders
+			err = u.symlinks[fullPath].Crawl()
 			if err != nil {
-				return nil, errors.Wrap(err, "add symlink")
-			}
-			if stat == nil {
-				return nil, nil
+				return nil, errors.Wrap(err, "crawl symlink")
 			}
+		}
+	} else if err != nil {
+		u.sync.log.Debugf("Error in lstat %s: %v", fullPath, err)
+		return nil, nil
+	} else if stat == nil {
+		return nil, nil
+	}
 
-			// Only crawl if symlink wasn't there before and it is a directory
-			if !symlinkExists && stat.IsDir() {
-				// Crawl all linked files & folders
-				err = u.symlinks[fullpath].Crawl()
-				if err != nil {
-					return nil, errors.Wrap(err, "crawl symlink")
-				}
+	fileInfo := &FileInformation{
+		Name:           relativePath,
+		Mtime:          stat.ModTime().Unix(),
+		MtimeNano:      stat.ModTime().UnixNano(),
+		Size:           stat.Size(),
+		Mode:           stat.Mode(),
+		IsDirectory:    stat.IsDir(),
+		IsSymbolicLink: stat.Mode()&os.ModeSymlink != 0,
+	}
+
+	// should we upload the file?
+	if shouldUpload(u.sync, fileInfo, u.sync.log) {
+		// New Create Task
+		return []*FileInformation{fileInfo}, nil
+	} else if stat.IsDir() {
+		// if the change is a directory we walk the directory for other potential changes
+		files, err := ioutil.ReadDir(fullPath)
+		if err != nil {
+			// Remove symlinks
+			u.RemoveSymlinks(fullPath)
+
+			// Check if we should remove path remote
+			if shouldRemoveRemote(relativePath, u.sync) {
+				// New Remove Task
+				return []*FileInformation{
+					{
+						Name: relativePath,
+					},
+				}, nil
 			}
-		} else if err != nil {
-			u.sync.log.Debugf("Error in lstat %s: %v", fullpath, err)
-			return nil, nil
-		} else if stat == nil {
+
 			return nil, nil
 		}
 
-		fileInfo := &FileInformation{
-			Name:           relativePath,
-			Mtime:          stat.ModTime().Unix(),
-			MtimeNano:      stat.ModTime().UnixNano(),
-			Size:           stat.Size(),
-			Mode:           stat.Mode(),
-			IsDirectory:    stat.IsDir(),
-			IsSymbolicLink: stat.Mode()&os.ModeSymlink != 0,
-		}
-		if shouldUpload(u.sync, fileInfo, u.sync.log) {
-			// New Create Task
-			return fileInfo, nil
-		}
-	} else {
-		// Remove symlinks
-		u.RemoveSymlinks(fullpath)
+		changes := []*FileInformation{}
+		for _, f := range files {
+			newFullPath := filepath.Join(fullPath, f.Name())
+			newRelativePath := path.Join(relativePath, f.Name())
+			if fsutil.IsRecursiveSymlink(f, newFullPath) {
+				continue
+			}
 
-		// Check if we should remove path remote
-		if shouldRemoveRemote(relativePath, u.sync) {
-			// New Remove Task
-			return &FileInformation{
-				Name: relativePath,
-			}, nil
+			otherChanges, err := u.evaluateChange(newRelativePath, newFullPath)
+			if err != nil {
+				return nil, errors.Wrap(err, "evaluate change")
+			}
+
+			changes = append(changes, otherChanges...)
 		}
+
+		return changes, nil
 	}
 
 	return nil, nil