Skip to content

Commit 7116c96

Browse files
otoolepotoolep
authored
Merge pull request #21 from rqlite/merge-upstream-post-ser
Merge upstream post serialization
2 parents e728f55 + 44238f2 commit 7116c96

File tree

5 files changed

+198
-153
lines changed

5 files changed

+198
-153
lines changed

‎sqlite3.go‎

Lines changed: 0 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -969,54 +969,6 @@ func (c *SQLiteConn) begin(ctx context.Context) (driver.Tx, error) {
969969
return &SQLiteTx{c}, nil
970970
}
971971

972-
// Serialize returns a byte slice that is a serialization of the database.
973-
// If the database fails to serialize, a nil slice will be returned.
974-
//
975-
// See https://www.sqlite.org/c3ref/serialize.html
976-
func (c *SQLiteConn) Serialize(schema string) []byte {
977-
if schema == "" {
978-
schema = "main"
979-
}
980-
var zSchema *C.char
981-
zSchema = C.CString(schema)
982-
defer C.free(unsafe.Pointer(zSchema))
983-
984-
var sz C.sqlite3_int64
985-
ptr := C.sqlite3_serialize(c.db, zSchema, &sz, 0)
986-
if ptr == nil {
987-
return nil
988-
}
989-
defer C.sqlite3_free(unsafe.Pointer(ptr))
990-
return C.GoBytes(unsafe.Pointer(ptr), C.int(sz))
991-
}
992-
993-
// Deserialize causes the connection to disconnect from the current database
994-
// and then re-open as an in-memory database based on the contents of the
995-
// byte slice. If deserelization fails, error will contain the return code
996-
// of the underlying SQLite API call.
997-
//
998-
// When this function returns, the connection is referencing database
999-
// data in Go space, so the connection and associated database must be copied
1000-
// immediately if it is to be used further.
1001-
//
1002-
// See https://www.sqlite.org/c3ref/deserialize.html
1003-
func (c *SQLiteConn) Deserialize(b []byte, schema string) error {
1004-
if schema == "" {
1005-
schema = "main"
1006-
}
1007-
var zSchema *C.char
1008-
zSchema = C.CString(schema)
1009-
defer C.free(unsafe.Pointer(zSchema))
1010-
1011-
rc := C.sqlite3_deserialize(c.db, zSchema,
1012-
(*C.uint8_t)(unsafe.Pointer(&b[0])),
1013-
C.sqlite3_int64(len(b)), C.sqlite3_int64(len(b)), 0)
1014-
if rc != 0 {
1015-
return fmt.Errorf("deserialize failed with return %v", rc)
1016-
}
1017-
return nil
1018-
}
1019-
1020972
// Open database and return a new connection.
1021973
//
1022974
// A pragma can take either zero or one argument.

‎sqlite3_opt_serialize.go‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
// +build !libsqlite3 sqlite_serialize
2+
3+
package sqlite3
4+
5+
/*
6+
#ifndef USE_LIBSQLITE3
7+
#include <sqlite3-binding.h>
8+
#else
9+
#include <sqlite3.h>
10+
#endif
11+
#include <stdlib.h>
12+
#include <stdint.h>
13+
*/
14+
import "C"
15+
16+
import (
17+
"fmt"
18+
"math"
19+
"reflect"
20+
"unsafe"
21+
)
22+
23+
// Serialize returns a byte slice that is a serialization of the database.
24+
//
25+
// See https://www.sqlite.org/c3ref/serialize.html
26+
func (c *SQLiteConn) Serialize(schema string) ([]byte, error) {
27+
if schema == "" {
28+
schema = "main"
29+
}
30+
var zSchema *C.char
31+
zSchema = C.CString(schema)
32+
defer C.free(unsafe.Pointer(zSchema))
33+
34+
var sz C.sqlite3_int64
35+
ptr := C.sqlite3_serialize(c.db, zSchema, &sz, 0)
36+
if ptr == nil {
37+
return nil, fmt.Errorf("serialize failed")
38+
}
39+
defer C.sqlite3_free(unsafe.Pointer(ptr))
40+
41+
if sz > C.sqlite3_int64(math.MaxInt) {
42+
return nil, fmt.Errorf("serialized database is too large (%d bytes)", sz)
43+
}
44+
45+
cBuf := make([]byte, int(sz))
46+
sh := (*reflect.SliceHeader)(unsafe.Pointer(&cBuf))
47+
sh.Data = uintptr(unsafe.Pointer(ptr))
48+
49+
res := make([]byte, int(sz))
50+
copy(res, cBuf)
51+
return res, nil
52+
}
53+
54+
// Deserialize causes the connection to disconnect from the current database and
55+
// then re-open as an in-memory database based on the contents of the byte slice.
56+
//
57+
// See https://www.sqlite.org/c3ref/deserialize.html
58+
func (c *SQLiteConn) Deserialize(b []byte, schema string) error {
59+
if schema == "" {
60+
schema = "main"
61+
}
62+
var zSchema *C.char
63+
zSchema = C.CString(schema)
64+
defer C.free(unsafe.Pointer(zSchema))
65+
66+
67+
tmpBuf := (*C.uchar)(C.sqlite3_malloc64(C.sqlite3_uint64(len(b))))
68+
cBuf := make([]byte, len(b))
69+
sh := (*reflect.SliceHeader)(unsafe.Pointer(&cBuf))
70+
sh.Data = uintptr(unsafe.Pointer(tmpBuf))
71+
copy(cBuf, b)
72+
73+
rc := C.sqlite3_deserialize(c.db, zSchema, tmpBuf, C.sqlite3_int64(len(b)),
74+
C.sqlite3_int64(len(b)), C.SQLITE_DESERIALIZE_FREEONCLOSE)
75+
if rc != C.SQLITE_OK {
76+
return fmt.Errorf("deserialize failed with return %v", rc)
77+
}
78+
return nil
79+
}

‎sqlite3_opt_serialize_omit.go‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
// +build libsqlite3,!sqlite_serialize
2+
3+
package sqlite3
4+
5+
import (
6+
"errors"
7+
)
8+
9+
/*
10+
#cgo CFLAGS: -DSQLITE_OMIT_DESERIALIZE
11+
*/
12+
import "C"
13+
14+
func (c *SQLiteConn) Serialize(schema string) ([]byte, error) {
15+
return nil, errors.New("sqlite3: Serialize requires the sqlite_serialize build tag when using the libsqlite3 build tag")
16+
}
17+
18+
func (c *SQLiteConn) Deserialize(b []byte, schema string) error {
19+
return errors.New("sqlite3: Deserialize requires the sqlite_serialize build tag when using the libsqlite3 build tag")
20+
}

‎sqlite3_opt_serialize_test.go‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
// +build !libsqlite3 sqlite_serialize
2+
3+
package sqlite3
4+
5+
import (
6+
"context"
7+
"database/sql"
8+
"os"
9+
"testing"
10+
)
11+
12+
func TestSerializeDeserialize(t *testing.T) {
13+
// Connect to the source database.
14+
srcTempFilename := TempFilename(t)
15+
defer os.Remove(srcTempFilename)
16+
srcDb, err := sql.Open(driverName, srcTempFilename)
17+
if err != nil {
18+
t.Fatal("Failed to open the source database:", err)
19+
}
20+
defer srcDb.Close()
21+
err = srcDb.Ping()
22+
if err != nil {
23+
t.Fatal("Failed to connect to the source database:", err)
24+
}
25+
26+
// Connect to the destination database.
27+
destTempFilename := TempFilename(t)
28+
defer os.Remove(destTempFilename)
29+
destDb, err := sql.Open(driverName, destTempFilename)
30+
if err != nil {
31+
t.Fatal("Failed to open the destination database:", err)
32+
}
33+
defer destDb.Close()
34+
err = destDb.Ping()
35+
if err != nil {
36+
t.Fatal("Failed to connect to the destination database:", err)
37+
}
38+
39+
// Write data to source database.
40+
_, err = srcDb.Exec(`CREATE TABLE foo (name string)`)
41+
if err != nil {
42+
t.Fatal("Failed to create table in source database:", err)
43+
}
44+
_, err = srcDb.Exec(`INSERT INTO foo(name) VALUES("alice")`)
45+
if err != nil {
46+
t.Fatal("Failed to insert data into source database", err)
47+
}
48+
49+
// Serialize the source database
50+
srcConn, err := srcDb.Conn(context.Background())
51+
if err != nil {
52+
t.Fatal("Failed to get connection to source database:", err)
53+
}
54+
defer srcConn.Close()
55+
56+
var serialized []byte
57+
if err := srcConn.Raw(func(raw interface{}) error {
58+
var err error
59+
serialized, err = raw.(*SQLiteConn).Serialize("")
60+
return err
61+
}); err != nil {
62+
t.Fatal("Failed to serialize source database:", err)
63+
}
64+
srcConn.Close()
65+
66+
// Confirm that the destination database is initially empty.
67+
var destTableCount int
68+
err = destDb.QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table'").Scan(&destTableCount)
69+
if err != nil {
70+
t.Fatal("Failed to check the destination table count:", err)
71+
}
72+
if destTableCount != 0 {
73+
t.Fatalf("The destination database is not empty; %v table(s) found.", destTableCount)
74+
}
75+
76+
// Deserialize to destination database
77+
destConn, err := destDb.Conn(context.Background())
78+
if err != nil {
79+
t.Fatal("Failed to get connection to destination database:", err)
80+
}
81+
defer destConn.Close()
82+
83+
if err := destConn.Raw(func(raw interface{}) error {
84+
return raw.(*SQLiteConn).Deserialize(serialized, "")
85+
}); err != nil {
86+
t.Fatal("Failed to deserialize source database:", err)
87+
}
88+
destConn.Close()
89+
90+
// Confirm that destination database has been loaded correctly.
91+
var destRowCount int
92+
err = destDb.QueryRow(`SELECT COUNT(*) FROM foo`).Scan(&destRowCount)
93+
if err != nil {
94+
t.Fatal("Failed to count rows in destination database table", err)
95+
}
96+
if destRowCount != 1 {
97+
t.Fatalf("Destination table does not have the expected records")
98+
}
99+
}

‎sqlite3_test.go‎

Lines changed: 0 additions & 105 deletions
Original file line numberDiff line numberDiff line change
@@ -926,111 +926,6 @@ func TestTransaction(t *testing.T) {
926926
}
927927
}
928928

929-
func TestSerialize(t *testing.T) {
930-
d := SQLiteDriver{}
931-
932-
srcConn, err := d.Open(":memory:")
933-
if err != nil {
934-
t.Fatal("failed to get database connection:", err)
935-
}
936-
defer srcConn.Close()
937-
sqlite3conn := srcConn.(*SQLiteConn)
938-
939-
_, err = sqlite3conn.Exec(`CREATE TABLE foo (name string)`, nil)
940-
if err != nil {
941-
t.Fatal("failed to create table:", err)
942-
}
943-
_, err = sqlite3conn.Exec(`INSERT INTO foo(name) VALUES("alice")`, nil)
944-
if err != nil {
945-
t.Fatal("failed to insert record:", err)
946-
}
947-
948-
// Serialize the database to a file
949-
tempFilename := TempFilename(t)
950-
defer os.Remove(tempFilename)
951-
if err := ioutil.WriteFile(tempFilename, sqlite3conn.Serialize(""), 0644); err != nil {
952-
t.Fatalf("failed to write serialized database to disk")
953-
}
954-
955-
// Open the SQLite3 file, and test that contents are as expected.
956-
db, err := sql.Open("sqlite3", tempFilename)
957-
if err != nil {
958-
t.Fatal("failed to open database:", err)
959-
}
960-
defer db.Close()
961-
962-
rows, err := db.Query(`SELECT * FROM foo`)
963-
if err != nil {
964-
t.Fatal("failed to query database:", err)
965-
}
966-
defer rows.Close()
967-
968-
rows.Next()
969-
970-
var name string
971-
rows.Scan(&name)
972-
if exp, got := name, "alice"; exp != got {
973-
t.Errorf("Expected %s for fetched result, but got %s:", exp, got)
974-
}
975-
}
976-
977-
func TestDeserialize(t *testing.T) {
978-
var sqlite3conn *SQLiteConn
979-
d := SQLiteDriver{}
980-
tempFilename := TempFilename(t)
981-
defer os.Remove(tempFilename)
982-
983-
// Create source database on disk.
984-
conn, err := d.Open(tempFilename)
985-
if err != nil {
986-
t.Fatal("failed to open on-disk database:", err)
987-
}
988-
defer conn.Close()
989-
sqlite3conn = conn.(*SQLiteConn)
990-
_, err = sqlite3conn.Exec(`CREATE TABLE foo (name string)`, nil)
991-
if err != nil {
992-
t.Fatal("failed to create table:", err)
993-
}
994-
_, err = sqlite3conn.Exec(`INSERT INTO foo(name) VALUES("alice")`, nil)
995-
if err != nil {
996-
t.Fatal("failed to insert record:", err)
997-
}
998-
conn.Close()
999-
1000-
// Read database file bytes from disk.
1001-
b, err := ioutil.ReadFile(tempFilename)
1002-
if err != nil {
1003-
t.Fatal("failed to read database file on disk", err)
1004-
}
1005-
1006-
// Deserialize file contents into memory.
1007-
conn, err = d.Open(":memory:")
1008-
if err != nil {
1009-
t.Fatal("failed to open in-memory database:", err)
1010-
}
1011-
sqlite3conn = conn.(*SQLiteConn)
1012-
defer conn.Close()
1013-
if err := sqlite3conn.Deserialize(b, ""); err != nil {
1014-
t.Fatal("failed to deserialize database", err)
1015-
}
1016-
1017-
// Check database contents are as expected.
1018-
rows, err := sqlite3conn.Query(`SELECT * FROM foo`, nil)
1019-
if err != nil {
1020-
t.Fatal("failed to query database:", err)
1021-
}
1022-
if len(rows.Columns()) != 1 {
1023-
t.Fatal("incorrect number of columns returned:", len(rows.Columns()))
1024-
}
1025-
values := make([]driver.Value, 1)
1026-
rows.Next(values)
1027-
if v, ok := values[0].(string); !ok {
1028-
t.Fatalf("wrong type for value: %T", v)
1029-
} else if v != "alice" {
1030-
t.Fatal("wrong value returned", v)
1031-
}
1032-
}
1033-
1034929
func TestWAL(t *testing.T) {
1035930
tempFilename := TempFilename(t)
1036931
defer os.Remove(tempFilename)

0 commit comments

Comments
 (0)