Skip to content
Open
Show file tree
Hide file tree
Changes from 21 commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
c357e0c
Add Blob I/O
joriszwart Sep 1, 2022
6188ff5
Merge branch 'mattn:master' into master
joriszwart Sep 1, 2022
88d6cfd
Blob I/O: early return
joriszwart Sep 2, 2022
c6a9868
Blob I/O: use raw connection
joriszwart Sep 2, 2022
2866235
Blob I/O: doc
joriszwart Sep 2, 2022
867ec21
Blob I/O: unabbreviate offs → offset
joriszwart Sep 2, 2022
5583431
Blob I/O: use keys in struct init
joriszwart Sep 2, 2022
606d2d2
Blob I/O: clean up connection
joriszwart Sep 2, 2022
79df247
Blob I/O: move test data / connection
joriszwart Sep 2, 2022
dff49f9
Merge branch 'mattn:master' into master
joriszwart Sep 2, 2022
01dbf4b
Blob I/O: add write and seek
joriszwart Sep 3, 2022
893e77d
Blob I/O: format error
joriszwart Sep 7, 2022
7df73e0
Blob I/O: limit and expose blob size
joriszwart Sep 9, 2022
e519482
Blob I/O: correct usage of raw connection
joriszwart Sep 9, 2022
67dc006
Blob I/O: close db
joriszwart Sep 9, 2022
7f0a6fb
Blob I/O: use vfsdb
joriszwart Sep 11, 2022
5b53687
Blob I/O: insufficient space errors for Write, raw connection callback
joriszwart Sep 25, 2022
ef6fa5a
Blob I/O: helpful error
joriszwart Oct 19, 2022
fe0ed2e
Blob I/O: adhere to io.Writer
joriszwart Oct 26, 2022
cb853d0
Blob I/O: use :memory: for compatibility (SQLite <3.37.0)
joriszwart Nov 30, 2022
7d10860
Blob I/O: add extra checks after review
joriszwart Oct 8, 2023
3f8fe99
Blob I/O: fix TODO
joriszwart Oct 22, 2023
81fbf42
Merge branch 'mattn:master' into master
joriszwart May 18, 2025
a01c70a
Merge branch 'mattn:master' into master
joriszwart Sep 18, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
169 changes: 169 additions & 0 deletions blob_io.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
// Copyright (C) 2022 Yasuhiro Matsumoto <[email protected]>.
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package sqlite3

/*
#ifndef USE_LIBSQLITE3
#include "sqlite3-binding.h"
#else
#include <sqlite3.h>
#endif
#include <stdlib.h>
*/
import "C"

import (
"errors"
"fmt"
"io"
"math"
"runtime"
"unsafe"
)

// SQLiteBlob implements the SQLite Blob I/O interface.
type SQLiteBlob struct {
conn *SQLiteConn
blob *C.sqlite3_blob
size int
offset int
}

// Blob opens a blob.
//
// See https://www.sqlite.org/c3ref/blob_open.html for usage.
//
// Should only be used with conn.Raw.
func (conn *SQLiteConn) Blob(database, table, column string, rowid int64, flags int) (*SQLiteBlob, error) {
databaseptr := C.CString(database)
defer C.free(unsafe.Pointer(databaseptr))

tableptr := C.CString(table)
defer C.free(unsafe.Pointer(tableptr))

columnptr := C.CString(column)
defer C.free(unsafe.Pointer(columnptr))

var blob *C.sqlite3_blob
ret := C.sqlite3_blob_open(conn.db, databaseptr, tableptr, columnptr, C.longlong(rowid), C.int(flags), &blob)

if ret != C.SQLITE_OK {
return nil, conn.lastError()
}

size := int(C.sqlite3_blob_bytes(blob))
bb := &SQLiteBlob{conn: conn, blob: blob, size: size, offset: 0}

runtime.SetFinalizer(bb, (*SQLiteBlob).Close)

return bb, nil
}

// Read implements the io.Reader interface.
func (s *SQLiteBlob) Read(b []byte) (n int, err error) {
if s.offset >= s.size {
return 0, io.EOF
}

if len(b) == 0 {
return 0, nil
}

n = s.size - s.offset
if len(b) < n {
n = len(b)
}

p := &b[0]
ret := C.sqlite3_blob_read(s.blob, unsafe.Pointer(p), C.int(n), C.int(s.offset))
if ret != C.SQLITE_OK {
return 0, s.conn.lastError()
}

s.offset += n

return n, nil
}

// Write implements the io.Writer interface.
func (s *SQLiteBlob) Write(b []byte) (n int, err error) {
if len(b) == 0 {

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I recommend you create a copy of b and use that throughout. Slices may be mutated after passed to Write(). You might end up writing the first few bytes from the initial data and the rest from the new set when b is overwritten by the caller. In go only the slice header is passed by value. The underlying array is shared with the caller.

Something like:

tmp := make([]byte, len(b))
copy(tmp, b)

Then use tmp instead of b.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Doesn't making a copy defeat the purpose of streaming blobs?

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not really, as you'd only be copying the buffer (which is small - usually around 1024 bytes). But generally speaking you'd only have to copy if you need to guard against caller reuse of the buffer. This is usually done in logging writers that may want to return control back to the caller while the write operation against the logging backend is still ongoing.

I'm not sure that would apply here though (given that we want to wait for the write to happen and only then return), so I just added a comment as a "recommendation". Feel free to ignore me.

I haven't mentioned this, but thanks for reopening this PR!

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There is no need to make a copy of the slice, as sqlite3_blob_write is going to copy the data. And the caller is not allowed to modify the slice during the call the Write. (Even if they did, copy itself would be subject to a race condition anyway.)

return 0, nil
}

if s.offset >= s.size {
return 0, fmt.Errorf("sqlite3.SQLiteBlob.Write: insufficient space in %d-byte blob", s.size)
}

n = s.size - s.offset
if len(b) < n {
n = len(b)
}

if n != len(b) {
Copy link
Collaborator

@rittneje rittneje Mar 27, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Trying to remember - is there a reason not to do this check after the call to sqlite3_blob_write and only write what we can instead of nothing? I guess the current implementation is consistent with what sqlite3_blob_write internally does.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@joriszwart following up on this

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't know what to do. Sorry.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@rittneje can you help me out?

Copy link

@gabriel-samfira gabriel-samfira Sep 27, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This function implements io.Writer{}, which states:

https://pkg.go.dev/io#Writer

Write writes len(p) bytes from p to the underlying data stream. It returns the number of bytes written from p (0 <= n <= len(p)) and any error encountered that caused the write to stop early. Write must return a non-nil error if it returns n < len(p). Write must not modify the slice data, even temporarily.

The interesting bit of this is:

Write must return a non-nil error if it returns n < len(p).

So if b is partially written, then n should return the number of bytes written and a non nil error. But to be honest, the way it's written now, should be fine. I mean, partial writes are worse than no writes. At least the caller knows it erred and can retry using the same byte slice.

At least that's what I understand from the io.Writer docs.

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The only thing that would be nice to have is maybe an error that denotes not enough space. Something like ENOSPC on linux and ERROR_HANDLE_DISK_FULL on Windows (or if sqlite already has an error code for not enough space, to use that). But that's just a nit. It would help the caller when checking with errors.Is(err, NotEnoughSpacePlatformSpecificError).

I think sqlite has SQLITE_FULL and SQLITE_IOERR_DISKFULL.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

partial writes are worse than no writes

This is a subjective statement.

At least the caller knows it erred and can retry using the same byte slice.

If the error is because the blob is full, retrying is never going to work.

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You're right on both counts.

return 0, fmt.Errorf("sqlite3.SQLiteBlob.Write: insufficient space in %d-byte blob", s.size)
}

p := &b[0]
ret := C.sqlite3_blob_write(s.blob, unsafe.Pointer(p), C.int(n), C.int(s.offset))
if ret != C.SQLITE_OK {
return 0, s.conn.lastError()
}

s.offset += n

return n, nil
}

// Seek implements the io.Seeker interface.
func (s *SQLiteBlob) Seek(offset int64, whence int) (int64, error) {
if offset > math.MaxInt32 {
return 0, fmt.Errorf("sqlite3.SQLiteBlob.Seek: invalid offset %d", offset)
}

var abs int64
switch whence {
case io.SeekStart:
abs = offset
case io.SeekCurrent:
abs = int64(s.offset) + offset
case io.SeekEnd:
abs = int64(s.size) + offset
default:
return 0, fmt.Errorf("sqlite3.SQLiteBlob.Seek: invalid whence %d", whence)
}

if abs < 0 {
return 0, errors.New("sqlite.SQLiteBlob.Seek: negative position")
}

if abs > math.MaxInt32 || abs > int64(s.size) {
return 0, errors.New("sqlite3.SQLiteBlob.Seek: overflow position")
}

s.offset = int(abs)

return abs, nil
}

// Size returns the size of the blob.
func (s *SQLiteBlob) Size() int {
return s.size
}

// Close implements the io.Closer interface.
func (s *SQLiteBlob) Close() error {
ret := C.sqlite3_blob_close(s.blob)

s.blob = nil
runtime.SetFinalizer(s, nil)

if ret != C.SQLITE_OK {
return s.conn.lastError()
}

return nil
}
Loading