Merge pull request #49 from ipfs/kevina/4-to-5

4-to-5 migration improvments

Author: whyrusleeping

go-migrate/cli.go

@@ -7,11 +7,12 @@ import (
 )
 
 type Flags struct {
-	Force   bool
-	Revert  bool
-	Path    string // file path to migrate for fs based migrations
-	Verbose bool
-	Help    bool
+	Force    bool
+	Revert   bool
+	Path     string // file path to migrate for fs based migrations
+	Verbose  bool
+	Help     bool
+	NoRevert bool
 }
 
 func (f *Flags) Setup() {
@@ -20,6 +21,11 @@ func (f *Flags) Setup() {
 	flag.BoolVar(&f.Verbose, "verbose", false, "enable verbose logging")
 	flag.BoolVar(&f.Help, "help", false, "display help message")
 	flag.StringVar(&f.Path, "path", "", "file path to migrate for fs based migrations (required)")
+	flag.BoolVar(&f.NoRevert, "no-revert", false, "do not attempt to automatically revert on failure")
+}
+
+var SupportNoRevert = map[string]bool{
+	"4-to-5": true,
 }
 
 func (f *Flags) Parse() {
@@ -43,13 +49,17 @@ func Run(m Migration) error {
 
 	if !m.Reversible() {
 		if f.Revert {
-			return fmt.Errorf("migration %d is irreversible", m.Versions())
+			return fmt.Errorf("migration %s is irreversible", m.Versions())
 		}
 		if !f.Force {
-			return fmt.Errorf("migration %d is irreversible (use -f to proceed)", m.Versions())
+			return fmt.Errorf("migration %s is irreversible (use -f to proceed)", m.Versions())
 		}
 	}
 
+	if f.NoRevert && !SupportNoRevert[m.Versions()] {
+		return fmt.Errorf("migration %s does not support the '-no-revert' option", m.Versions())
+	}
+
 	if f.Revert {
 		return m.Revert(Options{
 			Flags:   f,

ipfs-4-to-5/migration/migration.go

@@ -53,10 +53,22 @@ func (m Migration) Apply(opts migrate.Options) error {
 	basepath := filepath.Join(opts.Path, "blocks")
 	ffspath := filepath.Join(opts.Path, "blocks-v4")
 	if err := os.Rename(basepath, ffspath); err != nil {
-		return err
+		if os.IsNotExist(err) {
+			fi, err2 := os.Stat(ffspath)
+			if err2 == nil && fi.IsDir() {
+				log.Log("... blocks already renamed to blocks-v4, continuing")
+				err = nil
+			}
+		}
+		if err != nil {
+			return err
+		}
 	}
 
 	revert1 := func(e error) error {
+		if opts.NoRevert {
+			return e
+		}
 		err := os.Rename(ffspath, basepath)
 		if err != nil {
 			log.Error(err)
@@ -66,20 +78,38 @@ func (m Migration) Apply(opts migrate.Options) error {
 
 	log.Log("> Upgrading datastore format to have sharding specification file")
 	if err := flatfs.UpgradeV0toV1(ffspath, 5); err != nil {
-		return revert1(err)
+		if os.IsExist(err) {
+			id, err2 := flatfs.ReadShardFunc(ffspath)
+			if err2 == nil && id.String() == flatfs.Prefix(5).String() {
+				log.Log("... datastore already has sharding specification file, continuing")
+				err = nil
+			}
+		}
+		if err != nil {
+			return revert1(err)
+		}
 	}
 
 	tempffs := filepath.Join(opts.Path, "blocks-v5")
 	log.Log("> creating a new flatfs datastore with new format")
 	if err := flatfs.Create(tempffs, flatfs.NextToLast(2)); err != nil {
-		if err2 := revertStep2(ffspath); err2 != nil {
-			log.Error(err2)
+		if err == flatfs.ErrDatastoreExists {
+			log.Log("... new flatfs datastore already exists continuing")
+			err = nil
+		}
+		if err != nil {
+			if err2 := revertStep2(ffspath); err2 != nil {
+				log.Error(err2)
+			}
+			return revert1(err)
 		}
-		return revert1(err)
 	}
 
 	revert3 := func(mainerr error) error {
 		log.Error("failed to convert flatfs datastore: %s", mainerr)
+		if opts.NoRevert {
+			return mainerr
+		}
 		log.Log("attempting to revert...")
 
 		if _, err := os.Stat(filepath.Join(ffspath, "SHARDING")); os.IsNotExist(err) {
@@ -114,6 +144,9 @@ func (m Migration) Apply(opts migrate.Options) error {
 	}
 
 	revert4 := func(mainerr error) error {
+		if opts.NoRevert {
+			return mainerr
+		}
 		if err := os.Mkdir(ffspath, 0755); err != nil {
 			log.Error("recreating flatfs directory: %s", err)
 			return err

sharness/t0080-migration-3-to-4.sh

@@ -105,6 +105,10 @@ test_expect_success "get pin lists" '
 
 test_kill_ipfs_daemon
 
+test_expect_success "'ipfs-3-to-4 -no-revert' fails" '
+	test_must_fail ipfs-3-to-4 -no-revert -path="$IPFS_PATH"
+'
+
 test_install_ipfs_nd "v0.4.3-dev"
 
 test_launch_ipfs_daemon

sharness/t0091-4-to-5-error-recovery.sh

@@ -0,0 +1,78 @@
+#!/bin/sh
+
+test_description="Test migration 4 to 5 woth -no-revert"
+
+. lib/test-lib.sh
+
+# setup vars for tests
+
+export IPFS_DIST_PATH="/ipfs/QmRgeXaMTu426NcVvTNywAR7HyFBFGhvRc9FuTtKx3Hfno"
+
+DEPTH=3
+NBDIR=3
+NBFILE=6
+PINTOTAL=20
+
+PINEACH=$(expr $PINTOTAL / 2)
+
+echo "DEPTH: $DEPTH"
+echo "NBDIR: $NBDIR"
+echo "NBFILE: $NBFILE"
+echo "PINTOTAL: $PINTOTAL"
+echo "PINEACH: $PINEACH"
+
+export GOPATH="$(pwd)/gopath"
+mkdir -p gopath/bin
+export PATH="../bin:$GOPATH/bin:$PATH"
+
+test_install_ipfs_nd "v0.4.4"
+
+test_init_ipfs_nd
+
+test_expect_success "make a couple files" '
+	rm -rf manyfiles &&
+	random-files -depth=$DEPTH -dirs=$NBDIR -files=$NBFILE manyfiles > filenames
+'
+
+test_expect_success "add a few files" '
+	ipfs add -r -q manyfiles | tee hashes
+'
+
+test_expect_success "get full ref list" '
+	ipfs refs local | sort > start_refs
+'
+test_expect_success "create a permission problem" '
+	RANDOM_DIR="`(cd "$IPFS_PATH"/blocks/ && ls -d CI??? | sort | head -3 | tail -1)`" &&
+	chmod 000 "$IPFS_PATH/blocks/$RANDOM_DIR"
+'
+
+test_expect_success "ipfs-4-to-5 -no-revert should fail" '
+	test_must_fail ipfs-4-to-5 -no-revert -path "$IPFS_PATH"
+'
+
+test_expect_success "ipfs-4-to-5 -no-revert leaves repo in inconsistent state" '
+	test -d "$IPFS_PATH"/blocks-v4 &&
+	test -d "$IPFS_PATH"/blocks-v5
+'
+
+test_expect_success "fix permission problem" '
+	chmod 700 "$IPFS_PATH/blocks-v4/$RANDOM_DIR"
+'
+
+test_expect_success "ipfs-4-to-5 -no-revert should be okay now" '
+	ipfs-4-to-5 -no-revert -path "$IPFS_PATH"
+'
+
+test_install_ipfs_nd "v0.4.5-pre2"
+
+test_expect_success "list all refs after migration" '
+	ipfs refs local | sort > after_refs
+'
+
+test_expect_success "refs look right" '
+	comm -23 start_refs after_refs > missing_refs &&
+	touch empty_refs_file &&
+	test_cmp missing_refs empty_refs_file
+'
+
+test_done