Merge pull request #53 from ipfs/fix/extra-files-flatfs

update to fix issue with extraneous files in flatfs during migration

Author: whyrusleeping

ipfs-4-to-5/go-ds-flatfs/convert.go

@@ -4,10 +4,13 @@
 package flatfs
 
 import (
+	"errors"
 	"fmt"
 	"io"
+	"log"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/ipfs/fs-repo-migrations/ipfs-2-to-3/Godeps/_workspace/src/github.com/jbenet/go-os-rename"
 	"github.com/ipfs/fs-repo-migrations/ipfs-4-to-5/go-datastore"
@@ -110,15 +113,45 @@ func Move(oldPath string, newPath string, out io.Writer) error {
 		if err != nil {
 			return err
 		}
-		if inf.IsDir() || fn == SHARDING_FN || fn == README_FN {
-			// we are an empty directory or generated file so just remove it
+		if inf.IsDir() {
+			indir, err := os.Open(oldPath)
+			if err != nil {
+				return err
+			}
+			defer indir.Close()
+
+			names, err := indir.Readdirnames(-1)
+			if err != nil {
+				return err
+			}
+
+			for _, n := range names {
+				p := filepath.Join(oldPath, n)
+				// part of unfinished write transaction
+				// remove it
+				if strings.HasPrefix(n, "put-") {
+					err := os.Remove(p)
+					if err != nil {
+						return err
+					}
+				} else {
+					return errors.New("unknown file in flatfs: " + p)
+				}
+			}
+
+			err = os.Remove(oldPath)
+			if err != nil {
+				return err
+			}
+		} else if fn == SHARDING_FN || fn == README_FN {
+			// generated file so just remove it
 			err := os.Remove(oldPath)
 			if err != nil {
 				return err
 			}
 		} else {
 			// else we found something unexpected, so to be safe just move it
-			fmt.Printf("found unexpected file in datastore directory: \"%s\", moving anyway", fn)
+			log.Printf("found unexpected file in datastore directory: \"%s\", moving anyway\n", fn)
 			newPath := filepath.Join(newDS.path, fn)
 			err := osrename.Rename(oldPath, newPath)
 			if err != nil {

ipfs-4-to-5/go-ds-flatfs/flatfs.go

@@ -7,18 +7,17 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"log"
 	"os"
-	"path"
 	"path/filepath"
 	"strings"
 	"time"
 
+	"github.com/ipfs/fs-repo-migrations/ipfs-2-to-3/Godeps/_workspace/src/github.com/jbenet/go-os-rename"
 	"github.com/ipfs/fs-repo-migrations/ipfs-4-to-5/go-datastore"
 	"github.com/ipfs/fs-repo-migrations/ipfs-4-to-5/go-datastore/query"
-	"github.com/ipfs/fs-repo-migrations/ipfs-2-to-3/Godeps/_workspace/src/github.com/jbenet/go-os-rename"
 )
 
-
 const (
 	extension = ".data"
 )
@@ -45,7 +44,7 @@ var (
 
 func Create(path string, fun *ShardIdV1) error {
 
-	err := os.Mkdir(path, 0777)
+	err := os.Mkdir(path, 0755)
 	if err != nil && !os.IsExist(err) {
 		return err
 	}
@@ -115,13 +114,13 @@ func (fs *Datastore) ShardStr() string {
 
 func (fs *Datastore) encode(key datastore.Key) (dir, file string) {
 	noslash := key.String()[1:]
-	dir = path.Join(fs.path, fs.getDir(noslash))
-	file = path.Join(dir, noslash+extension)
+	dir = filepath.Join(fs.path, fs.getDir(noslash))
+	file = filepath.Join(dir, noslash+extension)
 	return dir, file
 }
 
 func (fs *Datastore) decode(file string) (key datastore.Key, ok bool) {
-	if path.Ext(file) != extension {
+	if filepath.Ext(file) != extension {
 		return datastore.Key{}, false
 	}
 	name := file[:len(file)-len(extension)]
@@ -146,7 +145,7 @@ func (fs *Datastore) makeDir(dir string) error {
 }
 
 func (fs *Datastore) makeDirNoSync(dir string) error {
-	if err := os.Mkdir(dir, 0777); err != nil {
+	if err := os.Mkdir(dir, 0755); err != nil {
 		// EEXIST is safe to ignore here, that just means the prefix
 		// directory already existed.
 		if !os.IsExist(err) {
@@ -175,7 +174,7 @@ func (fs *Datastore) Put(key datastore.Key, value interface{}) error {
 			break
 		}
 
-		fmt.Printf("too many open files, retrying in %dms\n", 100*i)
+		log.Printf("too many open files, retrying in %dms\n", 100*i)
 		time.Sleep(time.Millisecond * 100 * time.Duration(i))
 	}
 	return err
@@ -430,7 +429,7 @@ func (fs *Datastore) walk(path string, reschan chan query.Result) error {
 
 		key, ok := fs.decode(fn)
 		if !ok {
-			fmt.Println("failed to decode entry in flatfs")
+			log.Println("failed to decode entry in flatfs: ", fn)
 			continue
 		}
 

sharness/t0090-migration-4-to-5.sh

@@ -87,6 +87,12 @@ test_expect_success "get pin lists" '
 	ipfs pin ls --type=indirect | sort > start_ind_pins
 '
 
+test_expect_success "put some unexpected files in the flatfs dir" '
+	echo "foo" > "$IPFS_PATH/blocks/CIQBE/put-123451" &&
+	echo "bar" > "$IPFS_PATH/blocks/CIQKA/put-123451" &&
+	echo "nonsense" > "$IPFS_PATH/blocks/badbad"
+'
+
 test_kill_ipfs_daemon
 
 test_install_ipfs_nd "v0.4.5-pre2"