Skip to content
Open
Show file tree
Hide file tree
Changes from 20 commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
c357e0c
Add Blob I/O
joriszwart Sep 1, 2022
6188ff5
Merge branch 'mattn:master' into master
joriszwart Sep 1, 2022
88d6cfd
Blob I/O: early return
joriszwart Sep 2, 2022
c6a9868
Blob I/O: use raw connection
joriszwart Sep 2, 2022
2866235
Blob I/O: doc
joriszwart Sep 2, 2022
867ec21
Blob I/O: unabbreviate offs → offset
joriszwart Sep 2, 2022
5583431
Blob I/O: use keys in struct init
joriszwart Sep 2, 2022
606d2d2
Blob I/O: clean up connection
joriszwart Sep 2, 2022
79df247
Blob I/O: move test data / connection
joriszwart Sep 2, 2022
dff49f9
Merge branch 'mattn:master' into master
joriszwart Sep 2, 2022
01dbf4b
Blob I/O: add write and seek
joriszwart Sep 3, 2022
893e77d
Blob I/O: format error
joriszwart Sep 7, 2022
7df73e0
Blob I/O: limit and expose blob size
joriszwart Sep 9, 2022
e519482
Blob I/O: correct usage of raw connection
joriszwart Sep 9, 2022
67dc006
Blob I/O: close db
joriszwart Sep 9, 2022
7f0a6fb
Blob I/O: use vfsdb
joriszwart Sep 11, 2022
5b53687
Blob I/O: insufficient space errors for Write, raw connection callback
joriszwart Sep 25, 2022
ef6fa5a
Blob I/O: helpful error
joriszwart Oct 19, 2022
fe0ed2e
Blob I/O: adhere to io.Writer
joriszwart Oct 26, 2022
cb853d0
Blob I/O: use :memory: for compatibility (SQLite <3.37.0)
joriszwart Nov 30, 2022
7d10860
Blob I/O: add extra checks after review
joriszwart Oct 8, 2023
3f8fe99
Blob I/O: fix TODO
joriszwart Oct 22, 2023
81fbf42
Merge branch 'mattn:master' into master
joriszwart May 18, 2025
a01c70a
Merge branch 'mattn:master' into master
joriszwart Sep 18, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
161 changes: 161 additions & 0 deletions blob_io.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
// Copyright (C) 2022 Yasuhiro Matsumoto <[email protected]>.
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package sqlite3

/*
#ifndef USE_LIBSQLITE3
#include "sqlite3-binding.h"
#else
#include <sqlite3.h>
#endif
#include <stdlib.h>
*/
import "C"

import (
"errors"
"fmt"
"io"
"math"
"runtime"
"unsafe"
)

// SQLiteBlob implements the SQLite Blob I/O interface.
type SQLiteBlob struct {
conn *SQLiteConn
blob *C.sqlite3_blob
size int
offset int
}

// Blob opens a blob.
//
// See https://www.sqlite.org/c3ref/blob_open.html for usage.
//
// Should only be used with conn.Raw.
func (conn *SQLiteConn) Blob(database, table, column string, rowid int64, flags int) (*SQLiteBlob, error) {
databaseptr := C.CString(database)
defer C.free(unsafe.Pointer(databaseptr))

tableptr := C.CString(table)
defer C.free(unsafe.Pointer(tableptr))

columnptr := C.CString(column)
defer C.free(unsafe.Pointer(columnptr))

var blob *C.sqlite3_blob
ret := C.sqlite3_blob_open(conn.db, databaseptr, tableptr, columnptr, C.longlong(rowid), C.int(flags), &blob)

if ret != C.SQLITE_OK {
return nil, conn.lastError()
}

size := int(C.sqlite3_blob_bytes(blob))
bb := &SQLiteBlob{conn: conn, blob: blob, size: size, offset: 0}

runtime.SetFinalizer(bb, (*SQLiteBlob).Close)

return bb, nil
}

// Read implements the io.Reader interface.
func (s *SQLiteBlob) Read(b []byte) (n int, err error) {
if s.offset >= s.size {
return 0, io.EOF
}

n = s.size - s.offset
if len(b) < n {
n = len(b)
}

p := &b[0]
ret := C.sqlite3_blob_read(s.blob, unsafe.Pointer(p), C.int(n), C.int(s.offset))
if ret != C.SQLITE_OK {
return 0, s.conn.lastError()
}

s.offset += n

return n, nil
}

// Write implements the io.Writer interface.
func (s *SQLiteBlob) Write(b []byte) (n int, err error) {
if s.offset >= s.size {
return 0, fmt.Errorf("sqlite3.SQLiteBlob.Write: insufficient space in %d-byte blob", s.size)
}

n = s.size - s.offset
if len(b) < n {
n = len(b)
}

if n != len(b) {
Copy link
Collaborator

@rittneje rittneje Mar 27, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Trying to remember - is there a reason not to do this check after the call to sqlite3_blob_write and only write what we can instead of nothing? I guess the current implementation is consistent with what sqlite3_blob_write internally does.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@joriszwart following up on this

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't know what to do. Sorry.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@rittneje can you help me out?

Copy link

@gabriel-samfira gabriel-samfira Sep 27, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This function implements io.Writer{}, which states:

https://pkg.go.dev/io#Writer

Write writes len(p) bytes from p to the underlying data stream. It returns the number of bytes written from p (0 <= n <= len(p)) and any error encountered that caused the write to stop early. Write must return a non-nil error if it returns n < len(p). Write must not modify the slice data, even temporarily.

The interesting bit of this is:

Write must return a non-nil error if it returns n < len(p).

So if b is partially written, then n should return the number of bytes written and a non nil error. But to be honest, the way it's written now, should be fine. I mean, partial writes are worse than no writes. At least the caller knows it erred and can retry using the same byte slice.

At least that's what I understand from the io.Writer docs.

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The only thing that would be nice to have is maybe an error that denotes not enough space. Something like ENOSPC on linux and ERROR_HANDLE_DISK_FULL on Windows (or if sqlite already has an error code for not enough space, to use that). But that's just a nit. It would help the caller when checking with errors.Is(err, NotEnoughSpacePlatformSpecificError).

I think sqlite has SQLITE_FULL and SQLITE_IOERR_DISKFULL.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

partial writes are worse than no writes

This is a subjective statement.

At least the caller knows it erred and can retry using the same byte slice.

If the error is because the blob is full, retrying is never going to work.

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You're right on both counts.

return 0, fmt.Errorf("sqlite3.SQLiteBlob.Write: insufficient space in %d-byte blob", s.size)
}

p := &b[0]
ret := C.sqlite3_blob_write(s.blob, unsafe.Pointer(p), C.int(n), C.int(s.offset))
if ret != C.SQLITE_OK {
return 0, s.conn.lastError()
}

s.offset += n

return n, nil
}

// Seek implements the io.Seeker interface.
func (s *SQLiteBlob) Seek(offset int64, whence int) (int64, error) {
if offset > math.MaxInt32 {
return 0, fmt.Errorf("sqlite3.SQLiteBlob.Seek: invalid offset %d", offset)
}

var abs int64
switch whence {
case io.SeekStart:
abs = offset
case io.SeekCurrent:
abs = int64(s.offset) + offset
case io.SeekEnd:
abs = int64(s.size) + offset
default:
return 0, fmt.Errorf("sqlite3.SQLiteBlob.Seek: invalid whence %d", whence)
}

if abs < 0 {
return 0, errors.New("sqlite.SQLiteBlob.Seek: negative position")
}

if abs > math.MaxInt32 || abs > int64(s.size) {
return 0, errors.New("sqlite3.SQLiteBlob.Seek: overflow position")
}

s.offset = int(abs)

return abs, nil
}

// Size returns the size of the blob.
func (s *SQLiteBlob) Size() int {
return s.size
}

// Close implements the io.Closer interface.
func (s *SQLiteBlob) Close() error {
ret := C.sqlite3_blob_close(s.blob)

s.blob = nil
runtime.SetFinalizer(s, nil)

if ret != C.SQLITE_OK {
return s.conn.lastError()
}

return nil
}
Loading