Give each migration its own binary (#116)

Fully isolated migrations

Each migration is a separate module that includes all dependencies vendored into it.

* Convert each ipfs-X-to-Y to a go module
* Rename each ipfs-X-to-Y to fs-repo-X-to-Y
* Move sharness tests into migration modules
* Update README.md to describe how to create migration
* Redesigned fs-repo-migrations binary to orchestrate, download, and execute the individual migrations
* The `fs-repo-migrations version` command prints latest migration available
* Added tools module for common migration code
* Updated sharness tests to use newer ipfs-update and test it for migrations

Co-authored-by: Adin Schmahmann <[email protected]>

Author: gammazero

.gitignore

@@ -1,6 +1,13 @@
-fs-repo-migrations
-ipfs-0-to-1/ipfs-0-to-1
-ipfs-1-to-2/ipfs-1-to-2
-ipfs-10-to-11/ipfs-10-to-11
-sharness/bin/*
 *~
+cmd/fs-repo-migrations/fs-repo-migrations
+sharness/bin/fs-repo-migrations
+sharness/bin/fs-repo-*-to-*
+sharness/bin/go-sleep
+sharness/bin/ipfs-update
+sharness/bin/iptb
+sharness/bin/pollEndpoint
+sharness/bin/random-files
+sharness/BUILD-OPTIONS
+sharness/lib/sharness/
+sharness/test-results/
+sharness/trash directory.*.sh/

Makefile

@@ -1,16 +1,43 @@
 GO111MODULE = on
 
-install:
-	go install -mod=vendor
-	@echo "fs-repo-migrations now installed, type 'fs-repo-migrations' to run"
+MIG_DIRS = $(shell ls -d fs-repo-*-to-*)
 
-test: test_go sharness
+.PHONY: all build clean cmd sharness test test_go
+
+all: build
+
+show: $(MIG_DIRS)
+	@echo "$(MIG_DIRS)"
+
+build: $(shell ls -d fs-repo-*-to-* | sed -e 's/fs-repo/build.fs-repo/') cmd
+	@echo OK
+
+build.%: MIGRATION=$*
+build.%:
+	make -C $(MIGRATION)
+
+cmd: fs-repo-migrations/fs-repo-migrations
 
-test_go:
-	go build -mod=vendor
-	go test -mod=vendor $(shell go list ./... | grep -v /gx/)
+fs-repo-migrations/fs-repo-migrations:
+	cd fs-repo-migrations && go build
 
 sharness:
 	make -C sharness
 
-.PHONY: test test_go sharness
+test: test_go sharness
+
+clean: $(shell ls -d fs-repo-*-to-* | sed -e 's/fs-repo/clean.fs-repo/')
+	@make -C sharness clean
+	@cd fs-repo-migrations && go clean
+	@echo OK
+
+clean.%: MIGRATION=$*
+clean.%:
+	make -C $(MIGRATION) clean
+
+test_go: $(shell ls -d fs-repo-*-to-* | sed -e 's/fs-repo/test_go.fs-repo/')
+	@echo OK
+
+test_go.%: MIGRATION=$*
+test_go.%:
+	@cd $(MIGRATION) && go test -mod=vendor

README.md

@@ -5,9 +5,9 @@
 [![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs)
 [![standard-readme compliant](https://img.shields.io/badge/standard--readme-OK-green.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme)
 
-> Migrations for the filesystem repository of ipfs clients
+> Migrations for the filesystem repository of ipfs nodes
 
-These are migrations for the filesystem repository of [ipfs](https://github.com/ipfs/ipfs) clients. This tool is written in Go, and developed alongside [go-ipfs](https://github.com/ipfs/go-ipfs), but it should work with any repo conforming to the [fs-repo specs](https://github.com/ipfs/specs/tree/master/repo/fs-repo).
+These are migrations for the filesystem repository of [ipfs](https://github.com/ipfs/ipfs) nodes. Each migration builds a separate binary that converts a repository to the next version.  The `fs-repo-migrations` is a tool that downloads individual migrations from the ipfs distribution site and applies them in sequence to migrate the ipfs repository to the target version.  This tool is written in Go, and developed alongside [go-ipfs](https://github.com/ipfs/go-ipfs).
 
 ## Table of Contents
 
@@ -30,24 +30,23 @@ make install
 
 ### When should I migrate
 
-When you want to upgrade go-ipfs to a new version, you may need to
-migrate.
-
-Here is the table showing which repo version corresponds to which
-go-ipfs version:
-
-ipfs repo version | go-ipfs versions
------------------ | ----------------
-                1 |  0.0.0 - 0.2.3
-                2 |  0.3.0 - 0.3.11
-                3 |  0.4.0 - 0.4.2
-                4 |  0.4.3 - 0.4.5
-                5 |  0.4.6 - 0.4.10
-                6 |  0.4.11 - 0.4.15
-                7 |  0.4.16 - 0.4.23
-                8 |  0.5.0 - 0.6.0
-                9 |  0.5.0 - 0.6.0
-                10 | 0.6.0 - current
+When you want to upgrade go-ipfs to a new version, you may need to migrate.
+
+Here is the table showing which repo version corresponds to which go-ipfs version:
+
+| ipfs repo version | go-ipfs versions |
+| ----------------: | :--------------- |
+|                 1 | 0.0.0 - 0.2.3.   |
+|                 2 | 0.3.0 - 0.3.11   |
+|                 3 | 0.4.0 - 0.4.2    |
+|                 4 | 0.4.3 - 0.4.5    |
+|                 5 | 0.4.6 - 0.4.10   |
+|                 6 | 0.4.11 - 0.4.15  |
+|                 7 | 0.4.16 - 0.4.23  |
+|                 8 | 0.5.0 - 0.6.0    |
+|                 9 | 0.5.0 - 0.6.0    |
+|                10 | 0.6.0 - 0.7.0    |
+|                11 | 0.8.0 - current  |
 
 ### How to Run Migrations
 
@@ -62,9 +61,15 @@ Migrations are one of those things that can be extremely painful on users. At th
 - Frozen. After the tool is written, all code must be frozen and vendored.
 - To Spec. The tools must conform to the spec.
 
+#### Build and Test
+
+To create a new migration, create a go module in a directory named `fs-repo-X-to-Y`, where `X` is the repo "from" version and `Y` the repo "to" version.  Vendor the module's dependencies. The build tooling will find this module and build the migration binary.
+
+If the migration directory contains a subdirectory named `sharness`, tests contained in it are run using the sharness test tool. Tests must be named `tNNNN-*.sh`, where NNNN is a 4-digit sequence number.
+
 ### Dependencies
 
-Dependencies must be vendored independently for each migration. Unfortunately, dependencies _must not_ be vendored using go modules because we need to support multiple versions of the same dependency (for different migrations). 
+Dependencies must be vendored independently for each migration. Each migration is a separate go module with its own `vendor` directory (created with `go mod vendor` for that migration).  All migrations are built using `go build -mod=vendor` to ensure dependencies come from the module's `vendor` directory.
 
 ## Contribute
 

fs-repo-0-to-1/.gitignore

@@ -0,0 +1 @@
+fs-repo-0-to-1

fs-repo-0-to-1/Makefile

@@ -0,0 +1,7 @@
+.PHONY: build clean
+
+build:
+	go build -mod=vendor
+
+clean:
+	go clean

fs-repo-0-to-1/go.mod

@@ -0,0 +1,5 @@
+module github.com/ipfs/fs-repo-migrations/fs-repo-0-to-1
+
+go 1.15
+
+require github.com/ipfs/fs-repo-migrations/tools v0.0.0-20210323144402-297a63449538

fs-repo-0-to-1/go.sum

@@ -0,0 +1,2 @@
+github.com/ipfs/fs-repo-migrations/tools v0.0.0-20210323144402-297a63449538 h1:8LgAkWkawtm+fmqziG/1q/JjctLgMdaySQOfFsZYvv0=
+github.com/ipfs/fs-repo-migrations/tools v0.0.0-20210323144402-297a63449538/go.mod h1:fADeaHKxwS+SKhc52rsL0P1MUcnyK31a9AcaG0KcfY8=

fs-repo-0-to-1/main.go

@@ -0,0 +1,11 @@
+package main
+
+import (
+	mg0 "github.com/ipfs/fs-repo-migrations/fs-repo-0-to-1/migration"
+	migrate "github.com/ipfs/fs-repo-migrations/tools/go-migrate"
+)
+
+func main() {
+	m := &mg0.Migration{}
+	migrate.Main(m)
+}

fs-repo-0-to-1/migration/migration.go

@@ -0,0 +1,91 @@
+package mg0
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	lock "github.com/ipfs/fs-repo-migrations/fs-repo-0-to-1/repolock"
+	migrate "github.com/ipfs/fs-repo-migrations/tools/go-migrate"
+	mfsr "github.com/ipfs/fs-repo-migrations/tools/mfsr"
+)
+
+type Migration struct {
+}
+
+// Version is the int version number. This could be a string
+// in future versions
+func (m Migration) Versions() string {
+	return "0-to-1"
+}
+
+// Reversible returns whether this migration can be reverted.
+// Endeavor to make them all reversible. This is here only to warn users
+// in case this is not the case.
+func (m Migration) Reversible() bool {
+	return true
+}
+
+// Apply applies the migration in question.
+// This migration merely adds a version file.
+func (m Migration) Apply(opts migrate.Options) error {
+	repolk, err := lock.Lock(opts.Path)
+	if err != nil {
+		return err
+	}
+	defer repolk.Close()
+
+	repo := mfsr.RepoPath(opts.Path)
+
+	// first, check if there is a version file.
+	// if there is, bail out.
+	if v, err := repo.Version(); err == nil {
+		return fmt.Errorf("repo at %s is version %s (not 0)", opts.Path, v)
+	} else if !strings.Contains(err.Error(), "no version file in repo") {
+		return err
+	}
+
+	// add the version file
+	if err := repo.WriteVersion("1"); err != nil {
+		return err
+	}
+
+	if opts.Verbose {
+		fmt.Println("wrote version file")
+	}
+
+	fmt.Println("Migration 0 to 1 succeeded")
+	return nil
+}
+
+// Revert un-applies the migration in question. This should be best-effort.
+// Some migrations are definitively one-way. If so, return an error.
+func (m Migration) Revert(opts migrate.Options) error {
+	lk, err := lock.Lock(opts.Path)
+	if err != nil {
+		return err
+	}
+	defer lk.Close()
+
+	repo := mfsr.RepoPath(opts.Path)
+
+	repolk, err := lock.Lock(opts.Path)
+	if err != nil {
+		return err
+	}
+	defer repolk.Close()
+
+	if err := repo.CheckVersion("1"); err != nil {
+		return err
+	}
+
+	// remove the version file
+	if err := os.Remove(repo.VersionFile()); err != nil {
+		return err
+	}
+	if opts.Verbose {
+		fmt.Println("deleted version file")
+	}
+
+	return nil
+}

fs-repo-0-to-1/repolock/lock.go

@@ -0,0 +1,24 @@
+package lock
+
+import (
+	"fmt"
+	"io"
+	"path"
+
+	"github.com/ipfs/fs-repo-migrations/tools/lock"
+)
+
+var errRepoLock = `failed to acquire repo lock at %s/%s
+Is a daemon running? please stop it before running migration`
+
+// LockFile is the filename of the daemon lock, relative to config dir
+// TODO rename repo lock and hide name
+const LockFile = "daemon.lock"
+
+func Lock(confdir string) (io.Closer, error) {
+	c, err := lock.Lock(path.Join(confdir, LockFile))
+	if err != nil {
+		return nil, fmt.Errorf(errRepoLock, confdir, LockFile)
+	}
+	return c, nil
+}