Merge pull request #62 from ipfs/kevina/5to6

repo migration from 5 to 6

Author: whyrusleeping

.travis.yml

@@ -14,8 +14,7 @@ env:
   - TEST_VERBOSE=1
 
 script:
-  - cd sharness
-  - make
+  - make -k test
 
 # For docker containers
 

Makefile

@@ -1,3 +1,13 @@
 install:
 	go install
 	@echo "fs-repo-migrations now installed, type 'fs-repo-migrations' to run"
+
+test: test_go sharness
+
+test_go:
+	go test ./ipfs-5-to-6/... # go test ./... fails see #66
+
+sharness:
+	make -C sharness
+
+.PHONY: test test_go sharness

ipfs-5-to-6/main.go

@@ -0,0 +1,11 @@
+package main
+
+import (
+	migrate "github.com/ipfs/fs-repo-migrations/go-migrate"
+	mg5 "github.com/ipfs/fs-repo-migrations/ipfs-5-to-6/migration"
+)
+
+func main() {
+	m := mg5.Migration{}
+	migrate.Main(&m)
+}

ipfs-5-to-6/migration/ci_config.go

@@ -0,0 +1,57 @@
+package mg5
+
+import (
+	"strings"
+)
+
+// ciConfig is a case insensitive map to represent a IPFS
+// configuration when unmarshaled as a map[string]interface{} instead
+// of specialized structures.  It is needed becuase JSON library is
+// case insensitive when matching json values to struct fields
+type ciConfig struct {
+	conf   map[string]interface{}
+	fromLC map[string]string
+}
+
+func newCiConfig(conf map[string]interface{}) ciConfig {
+	return ciConfig{
+		conf:   conf,
+		fromLC: lcMap(conf),
+	}
+}
+
+func lcMap(conf map[string]interface{}) map[string]string {
+	fromLC := make(map[string]string)
+	for key, _ := range conf {
+		fromLC[strings.ToLower(key)] = key
+	}
+	return fromLC
+}
+
+// get gets a key, returns nil if the key doesn't exist
+func (c ciConfig) get(origkey string) interface{} {
+	lckey := strings.ToLower(origkey)
+	key, ok := c.fromLC[lckey]
+	if !ok {
+		return nil
+	}
+	return c.conf[key] // should not panic, key should exist
+}
+
+func (c ciConfig) set(origkey string, val interface{}) {
+	lckey := strings.ToLower(origkey)
+	key, ok := c.fromLC[lckey]
+	if ok {
+		c.conf[key] = val
+	}
+	c.fromLC[lckey] = origkey
+	c.conf[origkey] = val
+}
+
+func (c ciConfig) del(origkey string) {
+	lckey := strings.ToLower(origkey)
+	key, ok := c.fromLC[lckey]
+	if ok {
+		delete(c.conf, key)
+	}
+}

ipfs-5-to-6/migration/config_conv.go

@@ -0,0 +1,158 @@
+package mg5
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"reflect"
+)
+
+// convFunc does an inplace conversion of the "datastore"
+// configuration (represted as a map[string]interface{}) from one
+// version to another
+type convFunc func(ds ciConfig) error
+
+// convertFile converts a config file from one version to another, the
+// converted config is stored in
+func convertFile(orig string, new string, convFunc convFunc) (ciConfig, error) {
+	in, err := os.Open(orig)
+	if err != nil {
+		return ciConfig{}, err
+	}
+	out, err := os.Create(new)
+	if err != nil {
+		return ciConfig{}, err
+	}
+	return convert(in, out, convFunc)
+}
+
+// convert converts the config from one version to another, returns
+// the converted config as a map[string]interface{}
+func convert(in io.Reader, out io.Writer, convFunc convFunc) (ciConfig, error) {
+	data, err := ioutil.ReadAll(in)
+	if err != nil {
+		return ciConfig{}, err
+	}
+	confMap := make(map[string]interface{})
+	err = json.Unmarshal(data, &confMap)
+	if err != nil {
+		return ciConfig{}, err
+	}
+	conf := newCiConfig(confMap)
+	ds, _ := conf.get("datastore").(map[string]interface{})
+	if ds == nil {
+		return ciConfig{}, fmt.Errorf("Datastore field missing or of the wrong type")
+	}
+	err = convFunc(newCiConfig(ds))
+	if err != nil {
+		return ciConfig{}, err
+	}
+	fixed, err := json.MarshalIndent(confMap, "", "  ")
+	if err != nil {
+		return ciConfig{}, err
+	}
+	out.Write(fixed)
+	out.Write([]byte("\n"))
+	return conf, nil
+}
+
+func ver5to6(ds ciConfig) error {
+	noSyncVal := ds.get("nosync")
+	if noSyncVal == nil {
+		noSyncVal = interface{}(false)
+	}
+	noSync, ok := noSyncVal.(bool)
+	if !ok {
+		return fmt.Errorf("unsupported value for Datastore.NoSync fields: %v", noSyncVal)
+	}
+	ds.del("nosync")
+
+	dsTypeVal := ds.get("type")
+	if dsTypeVal == nil {
+		dsTypeVal = interface{}("")
+	}
+	dsType, ok := dsTypeVal.(string)
+	if !ok || (dsType != "default" && dsType != "leveldb" && dsType != "") {
+		return fmt.Errorf("unsupported value for Datastore.Type fields: %s", dsType)
+	}
+	ds.del("type")
+
+	// Path and Params never appear to have been used so just delete them
+	ds.del("path")
+	ds.del("params")
+
+	ds.set("Spec", datastoreSpec(!noSync))
+	return nil
+}
+
+func ver6to5(ds ciConfig) (err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			err = fmt.Errorf("incompatible config detected, downgrade not possible: %v", r.(error))
+		}
+	}()
+	spec := ds.get("Spec").(map[string]interface{})
+	if spec == nil {
+		return fmt.Errorf("Datastore.Spec field missing or not a json object, can't downgrade")
+	}
+	mounts, _ := spec["mounts"].([]interface{})
+	if mounts == nil {
+		return fmt.Errorf("Datastore.Spec.mounts field is missing or not an array")
+	}
+	var root, blocks interface{}
+	sync := true
+	for _, mount := range mounts {
+		switch mountpoint := mount.(map[string]interface{})["mountpoint"].(string); mountpoint {
+		case "/blocks":
+			sync = mount.(map[string]interface{})["child"].(map[string]interface{})["sync"].(bool)
+			blocks = mount
+		case "/":
+			root = mount
+		default:
+			return fmt.Errorf("unknown mountpoint: %s", mountpoint)
+		}
+	}
+	// normalize spec
+	spec["mounts"] = []interface{}{blocks, root}
+	expected := datastoreSpec(sync)
+	if !reflect.DeepEqual(spec, expected) {
+		return fmt.Errorf("Datastore.Spec field not of a supported value, can't downgrade")
+	}
+	ds.del("Spec")
+	ds.set("Type", "leveldb")
+	ds.set("Params", nil)
+	ds.set("NoSync", !sync)
+	println("ver6to5 done!")
+	return nil
+}
+
+func datastoreSpec(sync bool) map[string]interface{} {
+	return map[string]interface{}{
+		"type": "mount",
+		"mounts": []interface{}{
+			map[string]interface{}{
+				"mountpoint": "/blocks",
+				"type":       "measure",
+				"prefix":     "flatfs.datastore",
+				"child": map[string]interface{}{
+					"type":      "flatfs",
+					"path":      "blocks",
+					"sync":      sync,
+					"shardFunc": "/repo/flatfs/shard/v1/next-to-last/2",
+				},
+			},
+			map[string]interface{}{
+				"mountpoint": "/",
+				"type":       "measure",
+				"prefix":     "leveldb.datastore",
+				"child": map[string]interface{}{
+					"type":        "levelds",
+					"path":        "datastore",
+					"compression": "none",
+				},
+			},
+		},
+	}
+}