MVCC memory VFS. (#309)

Author: ncruces

go.mod

@@ -5,6 +5,7 @@ go 1.23.0
 toolchain go1.24.0
 
 require (
+	github.com/ncruces/aa v0.3.0
 	github.com/ncruces/julianday v1.0.0
 	github.com/ncruces/sort v0.1.5
 	github.com/tetratelabs/wazero v1.9.0

go.sum

@@ -2,6 +2,8 @@ github.com/dchest/siphash v1.2.3 h1:QXwFc8cFOR2dSa/gE6o/HokBMWtLUaNDVd+22aKHeEA=
 github.com/dchest/siphash v1.2.3/go.mod h1:0NvQU092bT0ipiFN++/rXm69QG9tVxLAlQHIXMPAkHc=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/ncruces/aa v0.3.0 h1:6NPcK3jsyPWRZBZWCyF7c4IzEQX4eJtkKJBA+IRKTkQ=
+github.com/ncruces/aa v0.3.0/go.mod h1:ctOw1LVqfuqzqg2S9LlR045bLAiXtaTiPMCL3zzl7Ik=
 github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
 github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
 github.com/ncruces/sort v0.1.5 h1:fiFWXXAqKI8QckPf/6hu/bGFwcEPrirIOFaJqWujs4k=

tests/parallel/parallel_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/ncruces/go-sqlite3/vfs"
 	_ "github.com/ncruces/go-sqlite3/vfs/adiantum"
 	"github.com/ncruces/go-sqlite3/vfs/memdb"
+	"github.com/ncruces/go-sqlite3/vfs/mvcc"
 	_ "github.com/ncruces/go-sqlite3/vfs/xts"
 )
 
@@ -98,6 +99,22 @@ func Test_memdb(t *testing.T) {
 	testIntegrity(t, name)
 }
 
+func Test_mvcc(t *testing.T) {
+	var iter int
+	if testing.Short() {
+		iter = 1000
+	} else {
+		iter = 5000
+	}
+
+	mvcc.Create("test.db", "")
+	name := "file:/test.db?vfs=mvcc" +
+		"&_pragma=busy_timeout(10000)"
+	createDB(t, name)
+	testParallel(t, name, iter)
+	testIntegrity(t, name)
+}
+
 func Test_adiantum(t *testing.T) {
 	if !vfs.SupportsFileLocking {
 		t.Skip("skipping without locks")
@@ -312,6 +329,16 @@ func Benchmark_memdb(b *testing.B) {
 	testParallel(b, name, b.N)
 }
 
+func Benchmark_mvcc(b *testing.B) {
+	mvcc.Create("test.db", "")
+	name := "file:/test.db?vfs=mvcc" +
+		"&_pragma=busy_timeout(10000)"
+	createDB(b, name)
+
+	b.ResetTimer()
+	testParallel(b, name, b.N)
+}
+
 func createDB(t testing.TB, name string) {
 	db, err := sqlite3.Open(name)
 	if err != nil {

vfs/mvcc/README.md

@@ -0,0 +1,9 @@
+# Go `mvcc` SQLite VFS
+
+This package implements the **EXPERIMENTAL** `"mvcc"` in-memory SQLite VFS.
+
+It has some benefits over the [`"memdb"`](../memdb/README.md) VFS:
+- panics do not corrupt a shared database;
+- single-writer not blocked by readers,
+- readers never block,
+- instant snapshots.

vfs/mvcc/api.go

@@ -0,0 +1,67 @@
+// Package mvcc implements the "mvcc" SQLite VFS.
+//
+// The "mvcc" [vfs.VFS] allows the same in-memory database to be shared
+// among multiple database connections in the same process,
+// as long as the database name begins with "/".
+//
+// Importing package mvcc registers the VFS:
+//
+//	import _ "github.com/ncruces/go-sqlite3/vfs/mvcc"
+package mvcc
+
+import (
+	"sync"
+
+	"github.com/ncruces/go-sqlite3/vfs"
+)
+
+func init() {
+	vfs.Register("mvcc", mvccVFS{})
+}
+
+var (
+	memoryMtx sync.Mutex
+	// +checklocks:memoryMtx
+	memoryDBs = map[string]*mvccDB{}
+)
+
+// Create creates a shared memory database,
+// using data as its initial contents.
+func Create(name string, data string) {
+	memoryMtx.Lock()
+	defer memoryMtx.Unlock()
+
+	db := &mvccDB{
+		refs: 1,
+		name: name,
+	}
+	memoryDBs[name] = db
+	if len(data) == 0 {
+		return
+	}
+	// Convert data from WAL/2 to rollback journal.
+	if len(data) >= 20 && (false ||
+		data[18] == 2 && data[19] == 2 ||
+		data[18] == 3 && data[19] == 3) {
+		db.data = db.data.
+			Put(0, data[:18]).
+			Put(18, "\001\001").
+			Put(20, data[20:])
+	} else {
+		db.data = db.data.Put(0, data)
+	}
+}
+
+// Delete deletes a shared memory database.
+func Delete(name string) {
+	memoryMtx.Lock()
+	defer memoryMtx.Unlock()
+	delete(memoryDBs, name)
+}
+
+// Snapshot stores a snapshot of database src into dst.
+func Snapshot(dst, src string) {
+	memoryMtx.Lock()
+	defer memoryMtx.Unlock()
+	memoryDBs[dst] = memoryDBs[src].fork()
+}

vfs/mvcc/example_test.go

@@ -0,0 +1,50 @@
+package mvcc_test
+
+import (
+	"database/sql"
+	_ "embed"
+	"fmt"
+	"log"
+
+	_ "github.com/ncruces/go-sqlite3/driver"
+	_ "github.com/ncruces/go-sqlite3/embed"
+	"github.com/ncruces/go-sqlite3/vfs/mvcc"
+)
+
+//go:embed testdata/test.db
+var testDB string
+
+func Example() {
+	mvcc.Create("test.db", testDB)
+
+	db, err := sql.Open("sqlite3", "file:/test.db?vfs=mvcc")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer db.Close()
+
+	_, err = db.Exec(`INSERT INTO users (id, name) VALUES (3, 'rust')`)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	rows, err := db.Query(`SELECT id, name FROM users`)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var id, name string
+		err = rows.Scan(&id, &name)
+		if err != nil {
+			log.Fatal(err)
+		}
+		fmt.Printf("%s %s\n", id, name)
+	}
+	// Output:
+	// 0 go
+	// 1 zig
+	// 2 whatever
+	// 3 rust
+}