cgosqlite: remove some cgo allocs

                             │    before    │                after                 │
                             │    sec/op    │    sec/op     vs base                │
    WALHookAndExec-8           2.354µ ±  7%   2.465µ ± 28%        ~ (p=0.812 n=10)
    Persist-8                  2.054µ ±  1%   2.026µ ±  3%   -1.36% (p=0.043 n=10)
    QueryRows100MixedTypes-8   42.79µ ±  1%   37.75µ ±  1%  -11.78% (p=0.000 n=10)
    EmptyExec-8                378.2n ±  0%   355.2n ±  1%   -6.09% (p=0.000 n=10)
    BeginTxNoop-8              9.445µ ± 38%   9.258µ ±  2%   -1.98% (p=0.002 n=10)
    geomean                    3.747µ         3.618µ         -3.45%

                             │    before    │                after                 │
                             │     B/op     │     B/op      vs base                │
    WALHookAndExec-8             56.00 ± 0%     48.00 ± 0%  -14.29% (p=0.000 n=10)
    Persist-8                    488.0 ± 0%     449.0 ± 0%   -7.99% (p=0.000 n=10)
    QueryRows100MixedTypes-8   7.422Ki ± 0%   4.281Ki ± 0%  -42.32% (p=0.000 n=10)
    EmptyExec-8                  40.00 ± 0%     32.00 ± 0%  -20.00% (p=0.000 n=10)
    BeginTxNoop-8                656.0 ± 0%     624.0 ± 0%   -4.88% (p=0.000 n=10)
    geomean                      352.6          285.2       -19.12%

                             │   before   │               after                │
                             │ allocs/op  │ allocs/op   vs base                │
    WALHookAndExec-8           3.000 ± 0%   2.000 ± 0%  -33.33% (p=0.000 n=10)
    Persist-8                  16.00 ± 0%   12.00 ± 0%  -25.00% (p=0.000 n=10)
    QueryRows100MixedTypes-8   609.0 ± 0%   207.0 ± 0%  -66.01% (p=0.000 n=10)
    EmptyExec-8                2.000 ± 0%   1.000 ± 0%  -50.00% (p=0.000 n=10)
    BeginTxNoop-8              17.00 ± 0%   13.00 ± 0%  -23.53% (p=0.000 n=10)
    geomean                    15.83        9.163       -42.12%

    Updates tailscale/corp#9919

Signed-off-by: Brad Fitzpatrick <[email protected]>

Author: bradfitz

cgosqlite/cgosqlite.go

@@ -90,10 +90,30 @@ type DB struct {
 	declTypes map[string]string
 }
 
+// cStmt is a wrapper around an sqlite3 *sqlite3_stmt. Except it's stored as
+// uintptr to avoid allocations due to poor interactions between cgo's pointer
+// checker and Go's escape analysis.
+//
+// The ptr method returns the value as a pointer, for call sites that haven't
+// yet or don't need the optimization. This lets us migrate incrementally.
+//
+// See http://go/corp/9919.
+type cStmt struct {
+	v C.handle_sqlite3_stmt
+}
+
+// cStmtFromPtr returns a cStmt from a C pointer.
+func cStmtFromPtr(p *C.sqlite3_stmt) cStmt {
+	return cStmt{v: C.handle_sqlite3_stmt(uintptr(unsafe.Pointer(p)))}
+}
+
+func (h cStmt) int() C.handle_sqlite3_stmt { return h.v }
+func (h cStmt) ptr() *C.sqlite3_stmt       { return (*C.sqlite3_stmt)(unsafe.Pointer(uintptr(h.v))) }
+
 // Stmt implements sqliteh.Stmt.
 type Stmt struct {
 	db    *DB
-	stmt  *C.sqlite3_stmt
+	stmt  cStmt
 	start C.struct_timespec
 
 	// used as scratch space when calling into cgo
@@ -199,60 +219,60 @@ func (db *DB) Prepare(query string, prepFlags sqliteh.PrepareFlags) (stmt sqlite
 		return nil, "", err
 	}
 	remainingQuery = query[len(query)-int(C.strlen(csqlTail)):]
-	return &Stmt{db: db, stmt: cstmt}, remainingQuery, nil
+	return &Stmt{db: db, stmt: cStmtFromPtr(cstmt)}, remainingQuery, nil
 }
 
 func (stmt *Stmt) DBHandle() sqliteh.DB {
-	cdb := C.sqlite3_db_handle(stmt.stmt)
+	cdb := C.sqlite3_db_handle(stmt.stmt.ptr())
 	if cdb != nil {
 		return &DB{db: cdb}
 	}
 	return nil
 }
 
 func (stmt *Stmt) SQL() string {
-	return C.GoString(C.sqlite3_sql(stmt.stmt))
+	return C.GoString(C.sqlite3_sql(stmt.stmt.ptr()))
 }
 
 func (stmt *Stmt) ExpandedSQL() string {
-	return C.GoString(C.sqlite3_expanded_sql(stmt.stmt))
+	return C.GoString(C.sqlite3_expanded_sql(stmt.stmt.ptr()))
 }
 
 func (stmt *Stmt) Reset() error {
-	return errCode(C.sqlite3_reset(stmt.stmt))
+	return errCode(C.sqlite3_reset(stmt.stmt.ptr()))
 }
 
 func (stmt *Stmt) Finalize() error {
-	return errCode(C.sqlite3_finalize(stmt.stmt))
+	return errCode(C.sqlite3_finalize(stmt.stmt.ptr()))
 }
 
 func (stmt *Stmt) ClearBindings() error {
-	return errCode(C.sqlite3_clear_bindings(stmt.stmt))
+	return errCode(C.sqlite3_clear_bindings(stmt.stmt.ptr()))
 }
 
 func (stmt *Stmt) ResetAndClear() (time.Duration, error) {
 	if stmt.start != (C.struct_timespec{}) {
 		stmt.duration = 0
-		err := errCode(C.reset_and_clear(stmt.stmt, &stmt.start, &stmt.duration))
+		err := errCode(C.reset_and_clear(stmt.stmt.int(), &stmt.start, &stmt.duration))
 		return time.Duration(stmt.duration), err
 	}
-	return 0, errCode(C.reset_and_clear(stmt.stmt, nil, nil))
+	return 0, errCode(C.reset_and_clear(stmt.stmt.int(), nil, nil))
 }
 
 func (stmt *Stmt) StartTimer() {
 	C.monotonic_clock_gettime(&stmt.start)
 }
 
 func (stmt *Stmt) ColumnDatabaseName(col int) string {
-	return C.GoString((*C.char)(unsafe.Pointer(C.sqlite3_column_database_name(stmt.stmt, C.int(col)))))
+	return C.GoString((*C.char)(unsafe.Pointer(C.sqlite3_column_database_name(stmt.stmt.ptr(), C.int(col)))))
 }
 
 func (stmt *Stmt) ColumnTableName(col int) string {
-	return C.GoString((*C.char)(unsafe.Pointer(C.sqlite3_column_table_name(stmt.stmt, C.int(col)))))
+	return C.GoString((*C.char)(unsafe.Pointer(C.sqlite3_column_table_name(stmt.stmt.ptr(), C.int(col)))))
 }
 
 func (stmt *Stmt) Step() (row bool, err error) {
-	res := C.sqlite3_step(stmt.stmt)
+	res := C.ts_sqlite3_step(stmt.stmt.int())
 	switch res {
 	case C.SQLITE_ROW:
 		return true, nil
@@ -265,7 +285,7 @@ func (stmt *Stmt) Step() (row bool, err error) {
 
 func (stmt *Stmt) StepResult() (row bool, lastInsertRowID, changes int64, d time.Duration, err error) {
 	stmt.rowid, stmt.changes, stmt.duration = 0, 0, 0
-	res := C.step_result(stmt.stmt, &stmt.rowid, &stmt.changes, &stmt.duration)
+	res := C.step_result(stmt.stmt.int(), &stmt.rowid, &stmt.changes, &stmt.duration)
 	lastInsertRowID = int64(stmt.rowid)
 	changes = int64(stmt.changes)
 	d = time.Duration(stmt.duration)
@@ -281,51 +301,51 @@ func (stmt *Stmt) StepResult() (row bool, lastInsertRowID, changes int64, d time
 }
 
 func (stmt *Stmt) BindDouble(col int, val float64) error {
-	return errCode(C.sqlite3_bind_double(stmt.stmt, C.int(col), C.double(val)))
+	return errCode(C.sqlite3_bind_double(stmt.stmt.ptr(), C.int(col), C.double(val)))
 }
 
 func (stmt *Stmt) BindInt64(col int, val int64) error {
-	return errCode(C.sqlite3_bind_int64(stmt.stmt, C.int(col), C.sqlite3_int64(val)))
+	return errCode(C.sqlite3_bind_int64(stmt.stmt.ptr(), C.int(col), C.sqlite3_int64(val)))
 }
 
 func (stmt *Stmt) BindNull(col int) error {
-	return errCode(C.sqlite3_bind_null(stmt.stmt, C.int(col)))
+	return errCode(C.sqlite3_bind_null(stmt.stmt.ptr(), C.int(col)))
 }
 
 func (stmt *Stmt) BindText64(col int, val string) error {
 	if len(val) == 0 {
-		return errCode(C.bind_text64_empty(stmt.stmt, C.int(col)))
+		return errCode(C.bind_text64_empty(stmt.stmt.int(), C.int(col)))
 	}
 	v := C.CString(val) // freed by sqlite
-	return errCode(C.bind_text64(stmt.stmt, C.int(col), v, C.sqlite3_uint64(len(val))))
+	return errCode(C.bind_text64(stmt.stmt.int(), C.int(col), v, C.sqlite3_uint64(len(val))))
 }
 
 func (stmt *Stmt) BindZeroBlob64(col int, n uint64) error {
-	return errCode(C.sqlite3_bind_zeroblob64(stmt.stmt, C.int(col), C.sqlite3_uint64(n)))
+	return errCode(C.sqlite3_bind_zeroblob64(stmt.stmt.ptr(), C.int(col), C.sqlite3_uint64(n)))
 }
 
 func (stmt *Stmt) BindBlob64(col int, val []byte) error {
 	var str *C.char
 	if len(val) > 0 {
 		str = (*C.char)(unsafe.Pointer(&val[0]))
 	}
-	return errCode(C.bind_blob64(stmt.stmt, C.int(col), str, C.sqlite3_uint64(len(val))))
+	return errCode(C.bind_blob64(stmt.stmt.int(), C.int(col), str, C.sqlite3_uint64(len(val))))
 }
 
 func (stmt *Stmt) BindParameterCount() int {
-	return int(C.sqlite3_bind_parameter_count(stmt.stmt))
+	return int(C.sqlite3_bind_parameter_count(stmt.stmt.ptr()))
 }
 
 func (stmt *Stmt) BindParameterName(col int) string {
-	cstr := C.sqlite3_bind_parameter_name(stmt.stmt, C.int(col))
+	cstr := C.sqlite3_bind_parameter_name(stmt.stmt.ptr(), C.int(col))
 	if cstr == nil {
 		return ""
 	}
 	return C.GoString(cstr)
 }
 
 func (stmt *Stmt) BindParameterIndex(name string) int {
-	return int(C.bind_parameter_index(stmt.stmt, name))
+	return int(C.bind_parameter_index(stmt.stmt.int(), name))
 }
 
 func (stmt *Stmt) BindParameterIndexSearch(name string) int {
@@ -340,51 +360,45 @@ func (stmt *Stmt) BindParameterIndexSearch(name string) int {
 }
 
 func (stmt *Stmt) ColumnCount() int {
-	return int(C.sqlite3_column_count(stmt.stmt))
+	return int(C.sqlite3_column_count(stmt.stmt.ptr()))
 }
 
 func (stmt *Stmt) ColumnName(col int) string {
-	return C.GoString(C.sqlite3_column_name(stmt.stmt, C.int(col)))
+	return C.GoString(C.sqlite3_column_name(stmt.stmt.ptr(), C.int(col)))
 }
 
 func (stmt *Stmt) ColumnText(col int) string {
-	str := (*C.char)(unsafe.Pointer(C.sqlite3_column_text(stmt.stmt, C.int(col))))
-	n := C.sqlite3_column_bytes(stmt.stmt, C.int(col))
+	str := (*C.char)(unsafe.Pointer(C.ts_sqlite3_column_text(stmt.stmt.int(), C.int(col))))
+	n := C.ts_sqlite3_column_bytes(stmt.stmt.int(), C.int(col))
 	if str == nil || n == 0 {
 		return ""
 	}
 	return C.GoStringN(str, n)
 }
 
 func (stmt *Stmt) ColumnBlob(col int) []byte {
-	res := C.sqlite3_column_blob(stmt.stmt, C.int(col))
+	res := C.sqlite3_column_blob(stmt.stmt.ptr(), C.int(col))
 	if res == nil {
 		return nil
 	}
-	n := int(C.sqlite3_column_bytes(stmt.stmt, C.int(col)))
-
-	slice := struct {
-		data unsafe.Pointer
-		len  int
-		cap  int
-	}{data: unsafe.Pointer(res), len: n, cap: n}
-	return *(*[]byte)(unsafe.Pointer(&slice))
+	n := int(C.ts_sqlite3_column_bytes(stmt.stmt.int(), C.int(col)))
+	return unsafe.Slice((*byte)(unsafe.Pointer(res)), n)
 }
 
 func (stmt *Stmt) ColumnDouble(col int) float64 {
-	return float64(C.sqlite3_column_double(stmt.stmt, C.int(col)))
+	return float64(C.ts_sqlite3_column_double(stmt.stmt.int(), C.int(col)))
 }
 
 func (stmt *Stmt) ColumnInt64(col int) int64 {
-	return int64(C.sqlite3_column_int64(stmt.stmt, C.int(col)))
+	return int64(C.ts_sqlite3_column_int64(stmt.stmt.int(), C.int(col)))
 }
 
 func (stmt *Stmt) ColumnType(col int) sqliteh.ColumnType {
-	return sqliteh.ColumnType(C.sqlite3_column_type(stmt.stmt, C.int(col)))
+	return sqliteh.ColumnType(C.ts_sqlite3_column_type(stmt.stmt.int(), C.int(col)))
 }
 
 func (stmt *Stmt) ColumnDeclType(col int) string {
-	cstr := C.sqlite3_column_decltype(stmt.stmt, C.int(col))
+	cstr := C.sqlite3_column_decltype(stmt.stmt.ptr(), C.int(col))
 	if cstr == nil {
 		return ""
 	}

cgosqlite/cgosqlite.h

@@ -1,25 +1,32 @@
+#include "stdint.h"
+
+// uintptr versions of sqlite3 pointer types, to avoid allocations
+// in cgo code. (go/corp/9919)
+typedef uintptr_t handle_sqlite3_stmt; // a *sqlite3_stmt
+typedef uintptr_t handle_sqlite3; // a *sqlite3 (DB conn)
+
 // Forward decls because the warnings make debugging painful.
 size_t _GoStringLen(_GoString_ s);
 const char *_GoStringPtr(_GoString_ s);
 
 // Helper methods to deal with int <-> pointer pain.
 
-static int bind_text64(sqlite3_stmt* stmt, int col, const char* str, sqlite3_uint64 len) {
-	return sqlite3_bind_text64(stmt, col, str, len, free, SQLITE_UTF8);
+static int bind_text64(handle_sqlite3_stmt stmt, int col, const char* str, sqlite3_uint64 len) {
+	return sqlite3_bind_text64((sqlite3_stmt*)(stmt), col, str, len, free, SQLITE_UTF8);
 }
 
-static int bind_text64_empty(sqlite3_stmt* stmt, int col) {
-	return sqlite3_bind_text64(stmt, col, "", 0, SQLITE_STATIC, SQLITE_UTF8);
+static int bind_text64_empty(handle_sqlite3_stmt stmt, int col) {
+	return sqlite3_bind_text64((sqlite3_stmt*)(stmt), col, "", 0, SQLITE_STATIC, SQLITE_UTF8);
 }
 
-static int bind_blob64(sqlite3_stmt* stmt, int col, char* str, sqlite3_uint64 n) {
-	return sqlite3_bind_blob64(stmt, col, str, n, SQLITE_TRANSIENT);
+static int bind_blob64(handle_sqlite3_stmt stmt, int col, char* str, sqlite3_uint64 n) {
+	return sqlite3_bind_blob64((sqlite3_stmt*)(stmt), col, str, n, SQLITE_TRANSIENT);
 }
 
 // We only need the Go string's memory for the duration of the call,
 // and the GC pins it for us if we pass the gostring_t to C, so we
 // do the conversion here instead of with C.CString.
-static int bind_parameter_index(sqlite3_stmt* stmt, _GoString_ s) {
+static int bind_parameter_index(handle_sqlite3_stmt stmt, _GoString_ s) {
 	size_t n = _GoStringLen(s);
 	const char *p = (const char *)_GoStringPtr(s);
 
@@ -29,7 +36,7 @@ static int bind_parameter_index(sqlite3_stmt* stmt, _GoString_ s) {
 		return 0;
 	}
 	memmove(zName, p, n);
-	return sqlite3_bind_parameter_index(stmt, zName);
+	return sqlite3_bind_parameter_index((sqlite3_stmt*)(stmt), zName);
 }
 
 static void monotonic_clock_gettime(struct timespec* t) {
@@ -45,7 +52,8 @@ static int64_t ns_since(const struct timespec t1)
 }
 
 // step_result combines several cgo calls to save overhead.
-static int step_result(sqlite3_stmt* stmt, sqlite3_int64* rowid, sqlite3_int64* changes, int64_t* duration_ns) {
+static int step_result(handle_sqlite3_stmt stmth, sqlite3_int64* rowid, sqlite3_int64* changes, int64_t* duration_ns) {
+	sqlite3_stmt* stmt = (sqlite3_stmt*)(stmth);
 	struct timespec t1;
 	if (duration_ns) {
 		monotonic_clock_gettime(&t1);
@@ -63,7 +71,8 @@ static int step_result(sqlite3_stmt* stmt, sqlite3_int64* rowid, sqlite3_int64*
 }
 
 // reset_and_clear combines two cgo calls to save overhead.
-static int reset_and_clear(sqlite3_stmt* stmt, struct timespec* start, int64_t* duration_ns) {
+static int reset_and_clear(handle_sqlite3_stmt stmth, struct timespec* start, int64_t* duration_ns) {
+	sqlite3_stmt* stmt = (sqlite3_stmt*)(stmth);
 	int ret = sqlite3_reset(stmt);
 	int ret2 = sqlite3_clear_bindings(stmt);
 	if (duration_ns) {
@@ -88,3 +97,27 @@ static int wal_callback_into_go(void *userData, sqlite3 *db, const char *dbName,
 static void ts_sqlite3_wal_hook_go(sqlite3* db) {
 	sqlite3_wal_hook(db, wal_callback_into_go, 0);
 }
+
+static int ts_sqlite3_step(handle_sqlite3_stmt stmt) {
+	return sqlite3_step((sqlite3_stmt*)(stmt));
+}
+
+static const unsigned char *ts_sqlite3_column_text(handle_sqlite3_stmt stmt, int iCol) {
+	return sqlite3_column_text((sqlite3_stmt*)(stmt), iCol);
+}
+
+static int ts_sqlite3_column_type(handle_sqlite3_stmt stmt, int iCol) {
+	return sqlite3_column_type((sqlite3_stmt*)(stmt), iCol);
+}
+
+static int ts_sqlite3_column_bytes(handle_sqlite3_stmt stmt, int iCol) {
+	return sqlite3_column_bytes((sqlite3_stmt*)(stmt), iCol);
+}
+
+static double ts_sqlite3_column_double(handle_sqlite3_stmt stmt, int iCol) {
+	return sqlite3_column_double((sqlite3_stmt*)(stmt), iCol);
+}
+
+static sqlite3_int64 ts_sqlite3_column_int64(handle_sqlite3_stmt stmt, int iCol) {
+	return sqlite3_column_int64((sqlite3_stmt*)(stmt), iCol);
+}