Merge pull request #96 from ipfs/migration/8-base32_encoded_keys

Migration 8-9 - Encoding key filenames with base32

Author: Stebalien

README.md

@@ -44,7 +44,9 @@ ipfs repo version | go-ipfs versions
                 4 |  0.4.3 - 0.4.5
                 5 |  0.4.6 - 0.4.10
                 6 |  0.4.11 - 0.4.15
-                7 |  0.4.16 - current
+                7 |  0.4.16 - 0.4.23
+                8 |  0.5.0 - current
+                9 |  0.5.0 - current
 
 ### How to Run Migrations
 

ipfs-8-to-9/main.go

@@ -0,0 +1,11 @@
+package main
+
+import (
+	migrate "github.com/ipfs/fs-repo-migrations/go-migrate"
+	mg8 "github.com/ipfs/fs-repo-migrations/ipfs-8-to-9/migration"
+)
+
+func main() {
+	m := mg8.Migration{}
+	migrate.Main(&m)
+}

ipfs-8-to-9/migration/migration.go

@@ -0,0 +1,142 @@
+package mg8
+
+import (
+	base32 "encoding/base32"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	migrate "github.com/ipfs/fs-repo-migrations/go-migrate"
+	mfsr "github.com/ipfs/fs-repo-migrations/mfsr"
+	log "github.com/ipfs/fs-repo-migrations/stump"
+)
+
+type Migration struct{}
+
+func (m Migration) Versions() string {
+	return "8-to-9"
+}
+
+func (m Migration) Reversible() bool {
+	return true
+}
+
+const keyFilenamePrefix = "key_"
+
+const keystoreRoot = "keystore"
+
+func isEncoded(name string) bool {
+	_, err := decode(name)
+	return err == nil
+}
+
+func encode(name string) (string, error) {
+	if name == "" {
+		return "", fmt.Errorf("key name must be at least one character")
+	}
+
+	encoder := base32.StdEncoding.WithPadding(base32.NoPadding)
+	encodedName := encoder.EncodeToString([]byte(name))
+	return keyFilenamePrefix + strings.ToLower(encodedName), nil
+}
+
+func decode(name string) (string, error) {
+	if !strings.HasPrefix(name, keyFilenamePrefix) {
+		return "", fmt.Errorf("key's filename has unexpected format")
+	}
+
+	nameWithoutPrefix := strings.ToUpper(name[len(keyFilenamePrefix):])
+	decoder := base32.StdEncoding.WithPadding(base32.NoPadding)
+	data, err := decoder.DecodeString(nameWithoutPrefix)
+
+	return string(data), err
+}
+
+func (m Migration) Apply(opts migrate.Options) error {
+	log.Verbose = opts.Verbose
+	log.Log("applying %s repo migration", m.Versions())
+
+	err := m.encodeDecode(
+		opts,
+		isEncoded, // skip if already encoded
+		encode,
+	)
+
+	if err != nil {
+		return err
+	}
+
+	err = mfsr.RepoPath(opts.Path).WriteVersion("9")
+	if err != nil {
+		log.Error("failed to update version file to 9")
+		return err
+	}
+
+	log.Log("updated version file")
+
+	return nil
+}
+
+func (m Migration) encodeDecode(opts migrate.Options, shouldApplyCodec func(string) bool, codec func(string) (string, error)) error {
+	keystoreRoot := filepath.Join(opts.Path, keystoreRoot)
+	fileInfos, err := ioutil.ReadDir(keystoreRoot)
+
+	if err != nil {
+		return err
+	}
+
+	for _, info := range fileInfos {
+		if info.IsDir() {
+			log.Log("skipping ", info.Name(), " as it is directory!")
+			continue
+		}
+
+		if shouldApplyCodec(info.Name()) {
+			log.Log("skipping ", info.Name(), ". Already in expected format!")
+			continue
+		}
+
+		log.VLog("Renaming key's filename: ", info.Name())
+		encodedName, err := codec(info.Name())
+		if err != nil {
+			return err
+		}
+
+		src := filepath.Join(keystoreRoot, info.Name())
+		dest := filepath.Join(keystoreRoot, encodedName)
+
+		if err := os.Rename(src, dest); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (m Migration) Revert(opts migrate.Options) error {
+	log.Verbose = opts.Verbose
+	log.Log("reverting migration")
+
+	err := m.encodeDecode(
+		opts,
+		func(name string) bool {
+			return !isEncoded(name) // skip if not encoded
+		},
+		decode,
+	)
+
+	if err != nil {
+		return err
+	}
+
+	err = mfsr.RepoPath(opts.Path).WriteVersion("8")
+	if err != nil {
+		log.Error("failed to update version file to 8")
+		return err
+	}
+
+	log.Log("updated version file")
+
+	return nil
+}

main.go

@@ -16,10 +16,12 @@ import (
 	mg4 "github.com/ipfs/fs-repo-migrations/ipfs-4-to-5/migration"
 	mg5 "github.com/ipfs/fs-repo-migrations/ipfs-5-to-6/migration"
 	mg6 "github.com/ipfs/fs-repo-migrations/ipfs-6-to-7/migration"
+	mg7 "github.com/ipfs/fs-repo-migrations/ipfs-7-to-8/migration"
+	mg8 "github.com/ipfs/fs-repo-migrations/ipfs-8-to-9/migration"
 	mfsr "github.com/ipfs/fs-repo-migrations/mfsr"
 )
 
-var CurrentVersion = 7
+var CurrentVersion = 9
 
 var migrations = []gomigrate.Migration{
 	&mg0.Migration{},
@@ -29,6 +31,8 @@ var migrations = []gomigrate.Migration{
 	&mg4.Migration{},
 	&mg5.Migration{},
 	&mg6.Migration{},
+	&mg7.Migration{},
+	&mg8.Migration{},
 }
 
 func GetIpfsDir() (string, error) {

sharness/Makefile

@@ -22,6 +22,7 @@ BINS += bin/ipfs-4-to-5
 BINS += bin/ipfs-5-to-6
 BINS += bin/ipfs-6-to-7
 BINS += bin/ipfs-7-to-8
+BINS += bin/ipfs-8-to-9
 BINS += bin/ipfs-update
 BINS += bin/random-files
 BINS += bin/go-sleep
@@ -38,6 +39,7 @@ IPFS_4_TO_5_SRC = ../ipfs-4-to-5
 IPFS_5_TO_6_SRC = ../ipfs-5-to-6
 IPFS_6_TO_7_SRC = ../ipfs-6-to-7
 IPFS_7_TO_8_SRC = ../ipfs-7-to-8
+IPFS_8_TO_9_SRC = ../ipfs-8-to-9
 
 # User might want to override those on the command line
 GOFLAGS =
@@ -125,6 +127,11 @@ bin/ipfs-7-to-8: $(call find_go_files, $(IPFS_7_TO_8_SRC)) BUILD-OPTIONS
 	@echo "*** installing $@ ***"
 	go build $(GOFLAGS) -o $@ $(IPFS_7_TO_8_SRC)
 
+bin/ipfs-8-to-9: $(call find_go_files, $(IPFS_8_TO_9_SRC)) BUILD-OPTIONS
+	@echo "*** installing $@ ***"
+	go build $(GOFLAGS) -o $@ $(IPFS_8_TO_9_SRC)
+
+
 BUILD-OPTIONS: FORCE
 	@bin/checkflags '$@' '$(GOFLAGS)' '*** new Go flags ***'
 

sharness/lib/test-lib.sh

@@ -263,7 +263,7 @@ test_init_ipfs_nd() {
 
 	test_expect_success "ipfs init succeeds" '
 		export IPFS_PATH="$(pwd)/.ipfs" &&
-		ipfs init -b=1024 > /dev/null
+		ipfs init -b=2048 > /dev/null
 	'
 
 	test_expect_success "prepare config -- mounting and bootstrap rm" '

sharness/t0030-simple-migration.sh

@@ -13,7 +13,7 @@ test_expect_success "'fs-repo-migrations -v' works" '
 '
 
 test_expect_success "'fs-repo-migrations -v' output looks good" '
-	echo "7" >expected &&
+	echo "9" >expected &&
 	test_cmp expected actual
 '
 
@@ -30,7 +30,7 @@ test_expect_success "'fs-repo-migrations -v' works" '
 '
 
 test_expect_success "'fs-repo-migrations -v' output looks good" '
-	echo "7" >expected &&
+	echo "9" >expected &&
 	test_cmp expected actual
 '
 

sharness/t0120-migration-7-to-8.sh

@@ -0,0 +1,96 @@
+#!/bin/sh
+
+test_description="Test migration 7 to 9"
+
+. lib/test-lib.sh
+
+# Dist specially built with a v0.5.0-dev-8to9pre1 release
+export IPFS_DIST_PATH="/ipfs/QmaaN2kipZfUpRSzwvUeG4Xi3yp1JJB294Vj8pnZ24hesF"
+export GOPATH="$(pwd)/gopath"
+mkdir -p gopath/bin
+export PATH="$(pwd)/../bin:$GOPATH/bin:$PATH"
+echo $IPFS_PATH
+
+test_install_ipfs_nd "v0.4.23"
+
+test_init_ipfs_nd
+
+# We need bootstrap addresses to migrate. These are defaults from v0.4.23.
+test_expect_success "add bootstrap addresses" '
+  test_config_set --json Bootstrap "[
+  \"/dnsaddr/bootstrap.libp2p.io/ipfs/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN\",
+  \"/dnsaddr/bootstrap.libp2p.io/ipfs/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa\",
+  \"/dnsaddr/bootstrap.libp2p.io/ipfs/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb\",
+  \"/dnsaddr/bootstrap.libp2p.io/ipfs/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt\",
+  \"/ip4/104.131.131.82/tcp/4001/ipfs/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ\",
+  \"/ip4/104.236.179.241/tcp/4001/ipfs/QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM\",
+  \"/ip4/128.199.219.111/tcp/4001/ipfs/QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu\",
+  \"/ip4/104.236.76.40/tcp/4001/ipfs/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64\",
+  \"/ip4/178.62.158.247/tcp/4001/ipfs/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd\",
+  \"/ip6/2604:a880:1:20::203:d001/tcp/4001/ipfs/QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM\",
+  \"/ip6/2400:6180:0:d0::151:6001/tcp/4001/ipfs/QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu\",
+  \"/ip6/2604:a880:800:10::4a:5001/tcp/4001/ipfs/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64\",
+  \"/ip6/2a03:b0c0:0:1010::23:1001/tcp/4001/ipfs/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd\"
+]"
+'
+
+test_expect_success "remember old bootstrap" '
+  ipfs config Bootstrap |  grep -E -o "Qm[a-zA-Z0-9]+" | sort > bootstrap_old
+'
+
+# no need to launch daemon as this works offline
+
+test_expect_success "add some keys to the keystore" '
+  ipfs key gen -t rsa thisISKey1 && ipfs key gen -t ed25519 key2
+'
+
+test_expect_success "ipfs key list" '
+  ipfs key list > key_list
+'
+
+test_expect_success "run migration 7 to 8" '
+  ipfs-7-to-8 -verbose -path="$IPFS_PATH" > migration_output8
+'
+
+test_expect_success "run migration 8 to 9" '
+  ipfs-8-to-9 -verbose -path="$IPFS_PATH" > migration_output9
+'
+
+test_expect_success "migration processed keys" '
+  grep "thisISKey1" migration_output9 &&
+  grep "key2" migration_output9
+'
+
+test_expect_success "migrated files exist" '
+  [ -f "${IPFS_PATH}/keystore/key_orugs42jknfwk6jr" ] &&
+  [ -f "${IPFS_PATH}/keystore/key_nnsxsmq" ]
+'
+
+test_install_ipfs_nd "v0.5.0-dev-8to9pre2"
+
+test_expect_success "ipfs key list is the same" '
+   ipfs key list > new_key_list
+   test_cmp key_list new_key_list
+'
+
+test_expect_success "migration revert to 8 succeeds" '
+  ipfs-8-to-9 -revert -verbose -path="$IPFS_PATH" > revert_output8
+'
+
+test_expect_success "migration revert to 7 succeeds" '
+  ipfs-7-to-8 -revert -verbose -path="$IPFS_PATH" > revert_output7
+'
+
+test_install_ipfs_nd "v0.4.23"
+
+test_expect_success "bootstrap addresses were reverted" '
+  ipfs config Bootstrap |  grep -E -o "Qm[a-zA-Z0-9]+" | sort > bootstrap_revert
+  test_cmp bootstrap_old bootstrap_revert
+'
+
+test_expect_success "ipfs key list is the same after revert" '
+  ipfs key list > revert_key_list
+  test_cmp key_list revert_key_list
+'
+
+test_done