Add 'import' command (#212)

Author: benbjohnson

cmd/litefs/import.go

@@ -0,0 +1,89 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/superfly/litefs/http"
+)
+
+// ImportCommand represents a command to import an existing SQLite database into a cluster.
+type ImportCommand struct {
+	// Target LiteFS URL
+	URL string
+
+	// Name of database on LiteFS cluster.
+	Name string
+
+	// SQLite database path to be imported
+	Path string
+}
+
+// NewImportCommand returns a new instance of ImportCommand.
+func NewImportCommand() *ImportCommand {
+	return &ImportCommand{
+		URL: DefaultURL,
+	}
+}
+
+// ParseFlags parses the command line flags & config file.
+func (c *ImportCommand) ParseFlags(ctx context.Context, args []string) (err error) {
+	fs := flag.NewFlagSet("litefs-import", flag.ContinueOnError)
+	fs.StringVar(&c.URL, "url", "http://localhost:20202", "LiteFS API URL")
+	fs.StringVar(&c.Name, "name", "", "database name")
+	fs.Usage = func() {
+		fmt.Println(`
+The import command will upload a SQLite database to a LiteFS cluster. If the
+named database doesn't exist, it will be created. If it does exist, it will be
+replaced. This command is safe to used on a live database.
+
+The database file is not validated for integrity by LiteFS. You can perform an
+integrity check first by running "PRAGMA integrity_check" from the SQLite CLI.
+
+Usage:
+
+	litefs import [arguments] PATH
+
+Arguments:
+`[1:])
+		fs.PrintDefaults()
+		fmt.Println("")
+	}
+	if err := fs.Parse(args); err != nil {
+		return err
+	} else if fs.NArg() == 0 {
+		fs.Usage()
+		return flag.ErrHelp
+	} else if fs.NArg() > 1 {
+		return fmt.Errorf("too many arguments")
+	}
+
+	// Copy first arg as database path.
+	c.Path = fs.Arg(0)
+
+	return nil
+}
+
+// Run executes the command.
+func (c *ImportCommand) Run(ctx context.Context) (err error) {
+	f, err := os.Open(c.Path)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = f.Close() }()
+
+	t := time.Now()
+
+	client := http.NewClient()
+	if err := client.Import(ctx, c.URL, c.Name, f); err != nil {
+		return err
+	}
+
+	// Notify user of success and elapsed time.
+	fmt.Printf("Import of database %q in %s\n", c.Name, time.Since(t))
+
+	return nil
+}

cmd/litefs/import_test.go

@@ -0,0 +1,104 @@
+package main_test
+
+import (
+	"context"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	main "github.com/superfly/litefs/cmd/litefs"
+	"github.com/superfly/litefs/internal/testingutil"
+)
+
+// Ensure a new, fresh database can be imported to a LiteFS server.
+func TestImportCommand_Create(t *testing.T) {
+	// Generate a database on the regular file system.
+	dsn := filepath.Join(t.TempDir(), "db")
+	db := testingutil.OpenSQLDB(t, dsn)
+	if _, err := db.Exec(`CREATE TABLE t (x)`); err != nil {
+		t.Fatal(err)
+	} else if _, err := db.Exec(`INSERT INTO t VALUES (100)`); err != nil {
+		t.Fatal(err)
+	} else if err := db.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Run a LiteFS mount.
+	m0 := runMountCommand(t, newMountCommand(t, t.TempDir(), nil))
+	waitForPrimary(t, m0)
+
+	// Import database into LiteFS.
+	cmd := main.NewImportCommand()
+	cmd.URL = m0.HTTPServer.URL()
+	cmd.Name = "my.db"
+	cmd.Path = dsn
+	if err := cmd.Run(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+
+	// Read from LiteFS mount.
+	db = testingutil.OpenSQLDB(t, filepath.Join(m0.Config.MountDir, "my.db"))
+	var x int
+	if err := db.QueryRow(`SELECT x FROM t`).Scan(&x); err != nil {
+		t.Fatal(err)
+	} else if got, want := x, 100; got != want {
+		t.Fatalf("x=%d, want %d", got, want)
+	}
+}
+
+// Ensure an existing database can be overwritten by an import.
+func TestImportCommand_Overwrite(t *testing.T) {
+	// Generate a database on the regular file system.
+	dsn := filepath.Join(t.TempDir(), "db")
+	dbx := testingutil.OpenSQLDB(t, dsn)
+	if _, err := dbx.Exec(`CREATE TABLE u (y)`); err != nil {
+		t.Fatal(err)
+	} else if _, err := dbx.Exec(`INSERT INTO u VALUES (100)`); err != nil {
+		t.Fatal(err)
+	} else if err := dbx.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Run an LiteFS mount.
+	m0 := runMountCommand(t, newMountCommand(t, t.TempDir(), nil))
+	waitForPrimary(t, m0)
+
+	// Generate data into the mount.
+	db := testingutil.OpenSQLDB(t, filepath.Join(m0.Config.MountDir, "db"))
+	if _, err := db.Exec(`CREATE TABLE t (x)`); err != nil {
+		t.Fatal(err)
+	}
+	for i := 0; i < 100; i++ {
+		if _, err := db.Exec(`INSERT INTO t VALUES (?)`, strings.Repeat("x", 256)); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// Overwrite database on LiteFS.
+	cmd := main.NewImportCommand()
+	cmd.URL = m0.HTTPServer.URL()
+	cmd.Name = "db"
+	cmd.Path = dsn
+	if err := cmd.Run(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+
+	// Read from LiteFS mount.
+	var y int
+	if err := db.QueryRow(`SELECT y FROM u`).Scan(&y); err != nil {
+		t.Fatal(err)
+	} else if got, want := y, 100; got != want {
+		t.Fatalf("y=%d, want %d", got, want)
+	}
+
+	// Reconnect and verify correctness.
+	if err := db.Close(); err != nil {
+		t.Fatal(err)
+	}
+	db = testingutil.OpenSQLDB(t, filepath.Join(m0.Config.MountDir, "db"))
+	if err := db.QueryRow(`SELECT y FROM u`).Scan(&y); err != nil {
+		t.Fatal(err)
+	} else if got, want := y, 100; got != want {
+		t.Fatalf("y=%d, want %d", got, want)
+	}
+}

cmd/litefs/main.go

@@ -24,6 +24,9 @@ var (
 	Commit  = ""
 )
 
+// DefaultURL refers to the LiteFS API on the local machine.
+const DefaultURL = "http://localhost:20202"
+
 func main() {
 	log.SetFlags(0)
 
@@ -43,11 +46,20 @@ func run(ctx context.Context, args []string) error {
 	}
 
 	switch cmd {
+	case "import":
+		c := NewImportCommand()
+		if err := c.ParseFlags(ctx, args); err != nil {
+			return err
+		}
+		return c.Run(ctx)
+
 	case "mount":
 		return runMount(ctx, args)
+
 	case "version":
 		fmt.Println(VersionString())
 		return nil
+
 	default:
 		if cmd == "" || cmd == "help" || strings.HasPrefix(cmd, "-") {
 			printUsage()
@@ -281,6 +293,7 @@ Usage:
 
 The commands are:
 
+	import       import a SQLite database into a LiteFS cluster
 	mount        mount the LiteFS FUSE file system
 	version      prints the version
 `[1:])