Skip to content

Commit c34a16e

Browse files
charlieviethcharlievieth
committed
sqlite3: reduce C to Go string conversions in SQLiteConn.{query,exec}
This commit fixes an issue where SQLiteConn.{exec,query} would use an exponential amount of memory when processing a prepared statement that consisted of multiple SQL statements ("stmt 1; stmt 2; ..."). Previously both exec/query used SQLiteConn.prepare() which converts the "tail" pointer set by sqlite3_prepare_v2() back into a Go string, which then has to be converted back to a C string for the next call to prepare() (assuming there are multiple SQL statements in the provided query). This commit fixes this by changing both exec and query to use the returned "tail" pointer for the next call to exec/query. It also changes prepare() to use pointer arithmetic to calculate the offset of the remaining "tail" portion of the query as a substring of the original query. This saves a call to C.GoString() and an allocation. Benchmarks: ``` goos: darwin goarch: arm64 pkg: github.com/mattn/go-sqlite3 │ base.10.txt │ new.10.txt │ │ sec/op │ sec/op vs base │ Suite/BenchmarkExec-10 1.351µ ± 1% 1.247µ ± 1% -7.74% (p=0.000 n=10) Suite/BenchmarkQuery-10 3.830µ ± 1% 3.558µ ± 1% -7.11% (p=0.000 n=10) Suite/BenchmarkParams-10 4.221µ ± 0% 4.228µ ± 1% ~ (p=1.000 n=10) Suite/BenchmarkStmt-10 2.906µ ± 1% 2.864µ ± 1% -1.45% (p=0.001 n=10) Suite/BenchmarkRows-10 149.1µ ± 4% 148.2µ ± 1% -0.61% (p=0.023 n=10) Suite/BenchmarkStmtRows-10 147.3µ ± 1% 145.6µ ± 0% -1.16% (p=0.000 n=10) Suite/BenchmarkExecStep-10 1898.9µ ± 3% 889.0µ ± 1% -53.18% (p=0.000 n=10) Suite/BenchmarkQueryStep-10 1848.0µ ± 1% 894.6µ ± 1% -51.59% (p=0.000 n=10) geomean 38.56µ 31.30µ -18.84% │ base.10.txt │ new.10.txt │ │ B/op │ B/op vs base │ Suite/BenchmarkExec-10 184.0 ± 0% 176.0 ± 0% -4.35% (p=0.000 n=10) Suite/BenchmarkQuery-10 864.0 ± 0% 856.0 ± 0% -0.93% (p=0.000 n=10) Suite/BenchmarkParams-10 1.289Ki ± 0% 1.281Ki ± 0% -0.61% (p=0.000 n=10) Suite/BenchmarkStmt-10 1.078Ki ± 0% 1.078Ki ± 0% ~ (p=1.000 n=10) ¹ Suite/BenchmarkRows-10 34.45Ki ± 0% 34.45Ki ± 0% -0.02% (p=0.000 n=10) Suite/BenchmarkStmtRows-10 34.40Ki ± 0% 34.40Ki ± 0% ~ (p=1.000 n=10) ¹ Suite/BenchmarkExecStep-10 5334.61Ki ± 0% 70.41Ki ± 0% -98.68% (p=0.000 n=10) Suite/BenchmarkQueryStep-10 5397.4Ki ± 0% 133.2Ki ± 0% -97.53% (p=0.000 n=10) geomean 17.06Ki 6.208Ki -63.62% ¹ all samples are equal │ base.10.txt │ new.10.txt │ │ allocs/op │ allocs/op vs base │ Suite/BenchmarkExec-10 13.00 ± 0% 12.00 ± 0% -7.69% (p=0.000 n=10) Suite/BenchmarkQuery-10 46.00 ± 0% 45.00 ± 0% -2.17% (p=0.000 n=10) Suite/BenchmarkParams-10 54.00 ± 0% 53.00 ± 0% -1.85% (p=0.000 n=10) Suite/BenchmarkStmt-10 49.00 ± 0% 49.00 ± 0% ~ (p=1.000 n=10) ¹ Suite/BenchmarkRows-10 2.042k ± 0% 2.041k ± 0% -0.05% (p=0.000 n=10) Suite/BenchmarkStmtRows-10 2.038k ± 0% 2.038k ± 0% ~ (p=1.000 n=10) ¹ Suite/BenchmarkExecStep-10 13.000k ± 0% 8.004k ± 0% -38.43% (p=0.000 n=10) Suite/BenchmarkQueryStep-10 11.013k ± 0% 6.017k ± 0% -45.36% (p=0.000 n=10) geomean 418.6 359.8 -14.04% ¹ all samples are equal ```
1 parent 3c0390b commit c34a16e

File tree

2 files changed

+94
-29
lines changed

2 files changed

+94
-29
lines changed

‎sqlite3.go

Lines changed: 73 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ package sqlite3
3030
#endif
3131
#include <stdlib.h>
3232
#include <string.h>
33+
#include <ctype.h>
3334
3435
#ifdef __CYGWIN__
3536
# include <errno.h>
@@ -90,6 +91,16 @@ _sqlite3_exec(sqlite3* db, const char* pcmd, long long* rowid, long long* change
9091
return rv;
9192
}
9293
94+
static const char *
95+
_trim_leading_spaces(const char *str) {
96+
if (str) {
97+
while (isspace(*str)) {
98+
str++;
99+
}
100+
}
101+
return str;
102+
}
103+
93104
#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY
94105
extern int _sqlite3_step_blocking(sqlite3_stmt *stmt);
95106
extern int _sqlite3_step_row_blocking(sqlite3_stmt* stmt, long long* rowid, long long* changes);
@@ -110,7 +121,11 @@ _sqlite3_step_row_internal(sqlite3_stmt* stmt, long long* rowid, long long* chan
110121
static int
111122
_sqlite3_prepare_v2_internal(sqlite3 *db, const char *zSql, int nBytes, sqlite3_stmt **ppStmt, const char **pzTail)
112123
{
113-
return _sqlite3_prepare_v2_blocking(db, zSql, nBytes, ppStmt, pzTail);
124+
int rv = _sqlite3_prepare_v2_blocking(db, zSql, nBytes, ppStmt, pzTail);
125+
if (pzTail) {
126+
*pzTail = _trim_leading_spaces(*pzTail);
127+
}
128+
return rv;
114129
}
115130
116131
#else
@@ -133,7 +148,11 @@ _sqlite3_step_row_internal(sqlite3_stmt* stmt, long long* rowid, long long* chan
133148
static int
134149
_sqlite3_prepare_v2_internal(sqlite3 *db, const char *zSql, int nBytes, sqlite3_stmt **ppStmt, const char **pzTail)
135150
{
136-
return sqlite3_prepare_v2(db, zSql, nBytes, ppStmt, pzTail);
151+
int rv = sqlite3_prepare_v2(db, zSql, nBytes, ppStmt, pzTail);
152+
if (pzTail) {
153+
*pzTail = _trim_leading_spaces(*pzTail);
154+
}
155+
return rv;
137156
}
138157
#endif
139158
@@ -858,25 +877,34 @@ func (c *SQLiteConn) Exec(query string, args []driver.Value) (driver.Result, err
858877
}
859878

860879
func (c *SQLiteConn) exec(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) {
880+
pquery := C.CString(query)
881+
op := pquery // original pointer
882+
defer C.free(unsafe.Pointer(op))
883+
884+
var stmtArgs []driver.NamedValue
885+
var tail *C.char
886+
s := new(SQLiteStmt) // escapes to the heap so reuse it
887+
defer s.finalize()
861888
start := 0
862889
for {
863-
s, err := c.prepare(ctx, query)
864-
if err != nil {
865-
return nil, err
890+
*s = SQLiteStmt{c: c} // reset
891+
rv := C._sqlite3_prepare_v2_internal(c.db, pquery, C.int(-1), &s.s, &tail)
892+
if rv != C.SQLITE_OK {
893+
return nil, c.lastError()
866894
}
895+
867896
var res driver.Result
868-
if s.(*SQLiteStmt).s != nil {
869-
stmtArgs := make([]driver.NamedValue, 0, len(args))
897+
if s.s != nil {
870898
na := s.NumInput()
871899
if len(args)-start < na {
872-
s.Close()
900+
s.finalize()
873901
return nil, fmt.Errorf("not enough args to execute query: want %d got %d", na, len(args))
874902
}
875903
// consume the number of arguments used in the current
876904
// statement and append all named arguments not
877905
// contained therein
878906
if len(args[start:start+na]) > 0 {
879-
stmtArgs = append(stmtArgs, args[start:start+na]...)
907+
stmtArgs = append(stmtArgs[:0], args[start:start+na]...)
880908
for i := range args {
881909
if (i < start || i >= na) && args[i].Name != "" {
882910
stmtArgs = append(stmtArgs, args[i])
@@ -886,23 +914,23 @@ func (c *SQLiteConn) exec(ctx context.Context, query string, args []driver.Named
886914
stmtArgs[i].Ordinal = i + 1
887915
}
888916
}
889-
res, err = s.(*SQLiteStmt).exec(ctx, stmtArgs)
917+
var err error
918+
res, err = s.exec(ctx, stmtArgs)
890919
if err != nil && err != driver.ErrSkip {
891-
s.Close()
920+
s.finalize()
892921
return nil, err
893922
}
894923
start += na
895924
}
896-
tail := s.(*SQLiteStmt).t
897-
s.Close()
898-
if tail == "" {
925+
s.finalize()
926+
if tail == nil || *tail == '\000' {
899927
if res == nil {
900928
// https://github.com/mattn/go-sqlite3/issues/963
901929
res = &SQLiteResult{0, 0}
902930
}
903931
return res, nil
904932
}
905-
query = tail
933+
pquery = tail
906934
}
907935
}
908936

@@ -919,22 +947,29 @@ func (c *SQLiteConn) Query(query string, args []driver.Value) (driver.Rows, erro
919947
}
920948

921949
func (c *SQLiteConn) query(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) {
950+
pquery := C.CString(query)
951+
op := pquery // original pointer
952+
defer C.free(unsafe.Pointer(op))
953+
954+
var stmtArgs []driver.NamedValue
955+
var tail *C.char
956+
s := new(SQLiteStmt) // escapes to the heap so reuse it
922957
start := 0
923958
for {
924-
stmtArgs := make([]driver.NamedValue, 0, len(args))
925-
s, err := c.prepare(ctx, query)
926-
if err != nil {
927-
return nil, err
959+
*s = SQLiteStmt{c: c, cls: true} // reset
960+
rv := C._sqlite3_prepare_v2_internal(c.db, pquery, C.int(-1), &s.s, &tail)
961+
if rv != C.SQLITE_OK {
962+
return nil, c.lastError()
928963
}
929-
s.(*SQLiteStmt).cls = true
964+
930965
na := s.NumInput()
931966
if len(args)-start < na {
932967
return nil, fmt.Errorf("not enough args to execute query: want %d got %d", na, len(args)-start)
933968
}
934969
// consume the number of arguments used in the current
935970
// statement and append all named arguments not contained
936971
// therein
937-
stmtArgs = append(stmtArgs, args[start:start+na]...)
972+
stmtArgs = append(stmtArgs[:0], args[start:start+na]...)
938973
for i := range args {
939974
if (i < start || i >= na) && args[i].Name != "" {
940975
stmtArgs = append(stmtArgs, args[i])
@@ -943,19 +978,18 @@ func (c *SQLiteConn) query(ctx context.Context, query string, args []driver.Name
943978
for i := range stmtArgs {
944979
stmtArgs[i].Ordinal = i + 1
945980
}
946-
rows, err := s.(*SQLiteStmt).query(ctx, stmtArgs)
981+
rows, err := s.query(ctx, stmtArgs)
947982
if err != nil && err != driver.ErrSkip {
948-
s.Close()
983+
s.finalize()
949984
return rows, err
950985
}
951986
start += na
952-
tail := s.(*SQLiteStmt).t
953-
if tail == "" {
987+
if tail == nil || *tail == '\000' {
954988
return rows, nil
955989
}
956990
rows.Close()
957-
s.Close()
958-
query = tail
991+
s.finalize()
992+
pquery = tail
959993
}
960994
}
961995

@@ -1818,8 +1852,11 @@ func (c *SQLiteConn) prepare(ctx context.Context, query string) (driver.Stmt, er
18181852
return nil, c.lastError()
18191853
}
18201854
var t string
1821-
if tail != nil && *tail != '\000' {
1822-
t = strings.TrimSpace(C.GoString(tail))
1855+
if tail != nil && *tail != 0 {
1856+
n := int(uintptr(unsafe.Pointer(tail))) - int(uintptr(unsafe.Pointer(pquery)))
1857+
if 0 <= n && n < len(query) {
1858+
t = strings.TrimSpace(query[n:])
1859+
}
18231860
}
18241861
ss := &SQLiteStmt{c: c, s: s, t: t}
18251862
runtime.SetFinalizer(ss, (*SQLiteStmt).Close)
@@ -1913,6 +1950,13 @@ func (s *SQLiteStmt) Close() error {
19131950
return nil
19141951
}
19151952

1953+
func (s *SQLiteStmt) finalize() {
1954+
if s.s != nil {
1955+
C.sqlite3_finalize(s.s)
1956+
s.s = nil
1957+
}
1958+
}
1959+
19161960
// NumInput return a number of parameters.
19171961
func (s *SQLiteStmt) NumInput() int {
19181962
return int(C.sqlite3_bind_parameter_count(s.s))

‎sqlite3_test.go

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2111,6 +2111,8 @@ var benchmarks = []testing.InternalBenchmark{
21112111
{Name: "BenchmarkStmt", F: benchmarkStmt},
21122112
{Name: "BenchmarkRows", F: benchmarkRows},
21132113
{Name: "BenchmarkStmtRows", F: benchmarkStmtRows},
2114+
{Name: "BenchmarkExecStep", F: benchmarkExecStep},
2115+
{Name: "BenchmarkQueryStep", F: benchmarkQueryStep},
21142116
}
21152117

21162118
func (db *TestDB) mustExec(sql string, args ...any) sql.Result {
@@ -2568,3 +2570,22 @@ func benchmarkStmtRows(b *testing.B) {
25682570
}
25692571
}
25702572
}
2573+
2574+
var largeSelectStmt = strings.Repeat("select 1;\n", 1_000)
2575+
2576+
func benchmarkExecStep(b *testing.B) {
2577+
for n := 0; n < b.N; n++ {
2578+
if _, err := db.Exec(largeSelectStmt); err != nil {
2579+
b.Fatal(err)
2580+
}
2581+
}
2582+
}
2583+
2584+
func benchmarkQueryStep(b *testing.B) {
2585+
var i int
2586+
for n := 0; n < b.N; n++ {
2587+
if err := db.QueryRow(largeSelectStmt).Scan(&i); err != nil {
2588+
b.Fatal(err)
2589+
}
2590+
}
2591+
}

0 commit comments

Comments
 (0)