Merge pull request #268 from achilleas-k/fix/add-slowness

Fix for add command seemingly hanging at the end

Author: mpsonntag

.travis.yml

@@ -8,6 +8,7 @@ services:
 env:
   global:
     - GO111MODULE=on
+    - GOPROXY=https://proxy.golang.org
 
 
 matrix:
@@ -22,15 +23,14 @@ matrix:
         homebrew:
           packages:
             - python3
+            - git-annex
+          update: true
   allow_failures:
     - go: tip
 
 before_install:
   - if [[ "${TRAVIS_OS_NAME}" != "osx" ]]; then
       pyenv global 3.7.1;
-    else
-      brew update;
-      brew install git-annex;
     fi
   - pip3 install --user pytest pyyaml
 

appveyor.yml

@@ -8,10 +8,11 @@ stack: go 1.12
 environment:
   SRCDIR: C:\gopath\src\github.com\G-Node\gin-cli
   GOPATH: C:\gopath
+  GO111MODULE: on
+  GOPROXY: https://proxy.golang.org
   BINDIR: bin
   # download dev build
   BUNDLEREPO: https://gin.g-node.org/achilleas/gin-cli-builds/raw/master
-  GIN_LOG_DIR: C:\gin-log
 
   matrix:
     # 32-bit
@@ -20,8 +21,6 @@ environment:
     # 64-bit
     - arch: "amd64"
 
-
-
 install:
   - set PATH=C:\Python37-x64\;%PATH%;%GOPATH%\bin;c:\gin-bundle\git\usr\bin
   # download latest stable gin-cli-windows-bundle
@@ -36,21 +35,7 @@ install:
 
 build_script:
   - cd %SRCDIR%
-  # download deps
-  - go get github.com/spf13/viper
-  - go get github.com/spf13/cobra
-  - go get github.com/howeyc/gopass
-  - go get golang.org/x/crypto/ssh
-  - go get github.com/G-Node/gin-core/gin
-  - go get github.com/G-Node/gin-repo/wire
-  - go get golang.org/x/crypto/ssh
-  - go get github.com/gogits/go-gogs-client
-  - go get github.com/fatih/color
-  - go get github.com/shibukawa/configdir
-  - go get github.com/bbrks/wrap
-  - go get github.com/docker/docker/pkg/term
-  - go get github.com/alecthomas/template
-  - go get github.com/dustin/go-humanize
+  - go get -v ./...
   - go vet ./...
   - set GOARCH=%arch%
   - go build -ldflags "-X main.gincliversion=APPVEYOR-%APPVEYOR_REPO_NAME%-%APPVEYOR_REPO_BRANCH% -X main.build=%APPVEYOR_BUILD_NUMBER% -X main.commit=%APPVEYOR_REPO_COMMIT%" -o %GOPATH%\bin\gin.exe .
@@ -74,6 +59,7 @@ test_script:
 deploy: off
 
 on_finish:
-  - appveyor PushArtifact %GIN_LOG_DIR%\gin.log
+  - appveyor PushArtifact %SRCDIR%\tests\log\gin.log
+  - appveyor PushArtifact %SRCDIR%\tests\log\runner.log
 # Uncomment next line to enable RDP
 # - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))

dorelease.py

@@ -200,16 +200,12 @@ def get_git_annex_for_windows():
     """
     Download the git annex for windows installer.
     """
-    win_git_annex_url = ("https://downloads.kitenet.net/git-annex/windows/"
-                         "current/git-annex-installer.exe")
-    return download(win_git_annex_url)
-    #
-    # git-annex version 7 is very broken on Windows. Until it is fixed, we are
-    # fixing the git-annex version (for Windows) to the last v6 version.
-    # This was obtained from the official git-annex website history:
-    # https://downloads.kitenet.net/.git/
-    # fname = os.path.join(DESTDIR, "downloads", "git-annex-windows-v6.exe")
-    # return fname
+    # win_git_annex_url = ("https://downloads.kitenet.net/git-annex/windows/"
+    #                      "current/git-annex-installer.exe")
+    # return download(win_git_annex_url)
+    # fixing windows installer version (again)
+    fname = os.path.join(DESTDIR, "downloads", "git-annex-installer.exe")
+    return fname
 
 
 def package_linux_plain(binfile):

ginclient/config/config.go

@@ -33,7 +33,7 @@ var (
 			Host:    "gin.g-node.org",
 			Port:    22,
 			User:    "git",
-			HostKey: "gin.g-node.org,141.84.41.216 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBE5IBgKP3nUryEFaACwY4N3jlqDx8Qw1xAxU2Xpt5V0p9RNefNnedVmnIBV6lA3n+9kT1OSbyqA/+SgsQ57nHo0=",
+			HostKey: "gin.g-node.org,141.84.41.219 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBE5IBgKP3nUryEFaACwY4N3jlqDx8Qw1xAxU2Xpt5V0p9RNefNnedVmnIBV6lA3n+9kT1OSbyqA/+SgsQ57nHo0=",
 		},
 	}
 

ginclient/repos.go

@@ -22,7 +22,7 @@ import (
 // High level functions for managing repositories.
 // These functions either end up performing web calls (using the web package) or git shell commands (using the git package).
 
-const unknownhostname = "(unknown)"
+const unknownhostname = "(unknownhost)"
 
 // Types
 
@@ -240,19 +240,35 @@ func Add(paths []string, addchan chan<- git.RepoFileStatus) {
 	}
 
 	if len(paths) > 0 {
+		gitaddpaths := make([]string, 0) // most times, this wont be used, so start with 0
+		statuschan := make(chan git.AnnexStatusRes)
+		go git.AnnexStatus(paths, statuschan)
+		for stat := range statuschan {
+			if stat.Status == "D" {
+				// deleted files match but weren't added
+				// this can happen when the annex filters don't match a file
+				// and it doesn't go through to get added to git
+				gitaddpaths = append(gitaddpaths, stat.File)
+			}
+		}
+
+		// Run git add on deleted files only
+		if len(gitaddpaths) > 0 {
+			gitaddchan := make(chan git.RepoFileStatus)
+			go git.Add(gitaddpaths, gitaddchan)
+			for addstat := range gitaddchan {
+				addstat.State = "Removing"
+				addchan <- addstat
+			}
+		}
+
 		// Run git annex add using exclusion filters
 		// Files matching filters are automatically added to git
 		annexaddchan := make(chan git.RepoFileStatus)
 		go git.AnnexAdd(paths, annexaddchan)
 		for addstat := range annexaddchan {
 			addchan <- addstat
 		}
-
-		gitaddchan := make(chan git.RepoFileStatus)
-		go git.Add(paths, gitaddchan)
-		for addstat := range gitaddchan {
-			addchan <- addstat
-		}
 	}
 }
 

gincmd/common.go

@@ -196,7 +196,9 @@ func printProgressWithBar(statuschan <-chan git.RepoFileStatus, nitems int) (fil
 		outline.Reset()
 		outline.WriteString(" ")
 		outappend(stat.State)
-		outappend(stat.FileName)
+		if stat.FileName != "" {
+			outappend(fmt.Sprintf("%q", stat.FileName))
+		}
 		if stat.Err == nil {
 			if stat.Progress == "100%" {
 				outappend(green("OK"))
@@ -247,7 +249,9 @@ func printProgressOutput(statuschan <-chan git.RepoFileStatus) (filesuccess map[
 			state = stat.State
 		}
 		outappend(stat.State)
-		outappend(stat.FileName)
+		if stat.FileName != "" {
+			outappend(fmt.Sprintf("%q", stat.FileName))
+		}
 		if stat.Err == nil {
 			if stat.Progress == "100%" {
 				outappend(green("OK"))
@@ -315,7 +319,7 @@ func determinePrintStyle(cmd *cobra.Command) printstyle {
 	case verboseOn && jsonOn:
 		Die("--verbose and --json cannot be used together")
 	case verboseOn:
-		git.JsonBool = false
+		git.RawMode = true
 		return psVerbose
 	case jsonOn:
 		return psJSON

git/annex.go

@@ -14,10 +14,14 @@ import (
 	"github.com/G-Node/gin-cli/git/shell"
 )
 
+// The following appears in the 'note' field when a file is added to git
+// instead of annex
+const addedToGitNote = "non-large file; adding content to git repository"
+
 // Git annex commands
 
-// Determine if json or normal command is used
-var JsonBool bool = true
+// RawMode disables --json output for annex commands
+var RawMode bool = false
 
 // Types (private)
 type annexAction struct {
@@ -262,11 +266,11 @@ func AnnexPush(paths []string, remote string, pushchan chan<- RepoFileStatus) {
 	}
 
 	var args []string
-	if JsonBool {
-		args = []string{"copy", "--json-progress", fmt.Sprintf("--to=%s", remote)}
-	} else {
-		args = []string{"copy", "--verbose", fmt.Sprintf("--to=%s", remote)}
+	outflag := "--json-progress"
+	if RawMode {
+		outflag = "--verbose"
 	}
+	args = []string{"copy", outflag, fmt.Sprintf("--to=%s", remote)}
 	if len(paths) == 0 {
 		paths = []string{"--all"}
 	}
@@ -299,7 +303,7 @@ func AnnexPush(paths []string, remote string, pushchan chan<- RepoFileStatus) {
 			// skip empty lines
 			continue
 		}
-		if !JsonBool {
+		if RawMode {
 			status.RawOutput = string(outline)
 			lineInput := cmd.Args
 			input := strings.Join(lineInput, " ")
@@ -393,7 +397,7 @@ func baseAnnexGet(cmdargs []string, getchan chan<- RepoFileStatus) {
 			continue
 		}
 
-		if !JsonBool {
+		if RawMode {
 			lineInput := cmd.Args
 			input := strings.Join(lineInput, " ")
 			status.RawInput = input
@@ -453,12 +457,11 @@ func baseAnnexGet(cmdargs []string, getchan chan<- RepoFileStatus) {
 // (git annex get)
 func AnnexGet(filepaths []string, getchan chan<- RepoFileStatus) {
 	defer close(getchan)
-	var cmdargs []string
-	if JsonBool {
-		cmdargs = append([]string{"get", "--json-progress"}, filepaths...)
-	} else {
-		cmdargs = append([]string{"get"}, filepaths...)
+	cmdargs := []string{"get"}
+	if !RawMode {
+		cmdargs = append(cmdargs, "--json-progress")
 	}
+	cmdargs = append(cmdargs, filepaths...)
 	baseAnnexGet(cmdargs, getchan)
 }
 
@@ -477,12 +480,12 @@ func AnnexGetKey(key string, getchan chan<- RepoFileStatus) {
 // (git annex drop)
 func AnnexDrop(filepaths []string, dropchan chan<- RepoFileStatus) {
 	defer close(dropchan)
-	var cmdargs []string
-	if JsonBool {
-		cmdargs = append([]string{"drop", "--json"}, filepaths...)
-	} else {
-		cmdargs = append([]string{"drop"}, filepaths...)
+	cmdargs := []string{"drop"}
+	if !RawMode {
+		cmdargs = append(cmdargs, "--json")
 	}
+	cmdargs = append(cmdargs, filepaths...)
+
 	cmd := AnnexCommand(cmdargs...)
 	err := cmd.Start()
 	if err != nil {
@@ -508,7 +511,7 @@ func AnnexDrop(filepaths []string, dropchan chan<- RepoFileStatus) {
 			continue
 		}
 
-		if !JsonBool {
+		if RawMode {
 			lineInput := cmd.Args
 			input := strings.Join(lineInput, " ")
 			status.RawInput = input
@@ -673,11 +676,9 @@ func AnnexLock(filepaths []string, lockchan chan<- RepoFileStatus) {
 		log.Write("No paths to lock. Nothing to do.")
 		return
 	}
-	var cmdargs []string
-	if JsonBool {
-		cmdargs = []string{"lock", "--json-error-messages"}
-	} else {
-		cmdargs = []string{"lock"}
+	cmdargs := []string{"lock"}
+	if !RawMode {
+		cmdargs = append(cmdargs, "--json-error-messages")
 	}
 
 	cmdargs = append(cmdargs, filepaths...)
@@ -700,7 +701,7 @@ func AnnexLock(filepaths []string, lockchan chan<- RepoFileStatus) {
 			// Empty line output. Ignore
 			continue
 		}
-		if !JsonBool {
+		if RawMode {
 			status.RawOutput = string(outline)
 			lineInput := cmd.Args
 			input := strings.Join(lineInput, " ")
@@ -756,11 +757,9 @@ func AnnexLock(filepaths []string, lockchan chan<- RepoFileStatus) {
 // (git annex unlock)
 func AnnexUnlock(filepaths []string, unlockchan chan<- RepoFileStatus) {
 	defer close(unlockchan)
-	var cmdargs []string
-	if JsonBool {
-		cmdargs = []string{"unlock", "--json"}
-	} else {
-		cmdargs = []string{"unlock"}
+	cmdargs := []string{"unlock"}
+	if !RawMode {
+		cmdargs = append(cmdargs, "--json")
 	}
 	cmdargs = append(cmdargs, filepaths...)
 	cmd := AnnexCommand(cmdargs...)
@@ -781,7 +780,7 @@ func AnnexUnlock(filepaths []string, unlockchan chan<- RepoFileStatus) {
 			// Empty line output. Ignore
 			continue
 		}
-		if !JsonBool {
+		if RawMode {
 			lineInput := cmd.Args
 			input := strings.Join(lineInput, " ")
 			status.RawInput = input
@@ -930,11 +929,9 @@ func AnnexAdd(filepaths []string, addchan chan<- RepoFileStatus) {
 		log.Write("No paths to add to annex. Nothing to do.")
 		return
 	}
-	var cmdargs []string
-	if JsonBool {
-		cmdargs = []string{"add", "--json"}
-	} else {
-		cmdargs = []string{"add"}
+	cmdargs := []string{"add"}
+	if !RawMode {
+		cmdargs = append(cmdargs, "--json")
 	}
 	exclargs := annexExclArgs()
 	if len(exclargs) > 0 {
@@ -954,21 +951,21 @@ func AnnexAdd(filepaths []string, addchan chan<- RepoFileStatus) {
 	var status RepoFileStatus
 	var addresult annexAction
 	// NOTE Can differentiate "git" and "annex" files from note in JSON struct
-	status.State = "Adding"
 	var filenames []string
 	for rerr = nil; rerr == nil; outline, rerr = cmd.OutReader.ReadBytes('\n') {
 		if len(outline) == 0 {
 			// Empty line output. Ignore
 			continue
 		}
-		if !JsonBool {
+		if RawMode {
 			status.RawOutput = string(outline)
 			lineInput := cmd.Args
 			input := strings.Join(lineInput, " ")
 			status.RawInput = input
 			addchan <- status
 			continue
 		}
+		addresult = annexAction{} // clear existing result
 		err := json.Unmarshal(outline, &addresult)
 		if err != nil || addresult.Command == "" {
 			// Couldn't parse output
@@ -983,7 +980,13 @@ func AnnexAdd(filepaths []string, addchan chan<- RepoFileStatus) {
 		status.RawInput = input
 		status.RawOutput = string(outline)
 		if addresult.Success {
-			log.Write("%s added to annex", addresult.File)
+			if addresult.Note == addedToGitNote {
+				status.State = "Added to git"
+				log.Write("%s added to git", addresult.File)
+			} else {
+				status.State = "Added to annex"
+				log.Write("%s added to annex", addresult.File)
+			}
 			status.Err = nil
 			filenames = append(filenames, status.FileName)
 		} else {