From b0f7da80406da121a515fb9580ac8a885f3c52e3 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Thu, 28 Jan 2021 21:50:49 -0500 Subject: [PATCH 01/39] Support serialize and deserialize --- sqlite3.go | 44 ++++++++++++++++++++ sqlite3_test.go | 105 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 149 insertions(+) diff --git a/sqlite3.go b/sqlite3.go index 552a2ab2..61a381ae 100644 --- a/sqlite3.go +++ b/sqlite3.go @@ -17,6 +17,7 @@ package sqlite3 #cgo CFLAGS: -DSQLITE_ENABLE_FTS3_PARENTHESIS #cgo CFLAGS: -DSQLITE_TRACE_SIZE_LIMIT=15 #cgo CFLAGS: -DSQLITE_OMIT_DEPRECATED +#cgo CFLAGS: -DSQLITE_ENABLE_DESERIALIZE #cgo CFLAGS: -DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1 #cgo CFLAGS: -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT #cgo CFLAGS: -Wno-deprecated-declarations @@ -905,6 +906,49 @@ func (c *SQLiteConn) begin(ctx context.Context) (driver.Tx, error) { return &SQLiteTx{c}, nil } +// Serialize returns a byte slice that is a serialization of the database. +// If the database fails to serialize, a nil slice will be returned. +// +// See https://www.sqlite.org/c3ref/serialize.html +func (c *SQLiteConn) Serialize(schema string) []byte { + if schema == "" { + schema = "main" + } + var zSchema *C.char + zSchema = C.CString(schema) + defer C.free(unsafe.Pointer(zSchema)) + + var sz C.sqlite3_int64 + ptr := C.sqlite3_serialize(c.db, zSchema, &sz, 0) + if ptr == nil { + return nil + } + return C.GoBytes(unsafe.Pointer(ptr), C.int(sz)) +} + +// Deserialize causes the connection to disconnect from the current database +// and then re-open as an in-memory database based on the contents of the +// byte slice. If deserelization fails, error will contain the return code +// of the underlying SQLite API call. +// +// See https://www.sqlite.org/c3ref/deserialize.html +func (c *SQLiteConn) Deserialize(b []byte, schema string) error { + if schema == "" { + schema = "main" + } + var zSchema *C.char + zSchema = C.CString(schema) + defer C.free(unsafe.Pointer(zSchema)) + + rc := C.sqlite3_deserialize(c.db, zSchema, + (*C.uint8_t)(unsafe.Pointer(&b[0])), + C.sqlite3_int64(len(b)), C.sqlite3_int64(len(b)), 0) + if rc != 0 { + return fmt.Errorf("deserialize failed with return %v", rc) + } + return nil +} + // Open database and return a new connection. // // A pragma can take either zero or one argument. diff --git a/sqlite3_test.go b/sqlite3_test.go index 878ec495..e5660c58 100644 --- a/sqlite3_test.go +++ b/sqlite3_test.go @@ -888,6 +888,111 @@ func TestTransaction(t *testing.T) { } } +func TestSerialize(t *testing.T) { + d := SQLiteDriver{} + + srcConn, err := d.Open(":memory:") + if err != nil { + t.Fatal("failed to get database connection:", err) + } + defer srcConn.Close() + sqlite3conn := srcConn.(*SQLiteConn) + + _, err = sqlite3conn.Exec(`CREATE TABLE foo (name string)`, nil) + if err != nil { + t.Fatal("failed to create table:", err) + } + _, err = sqlite3conn.Exec(`INSERT INTO foo(name) VALUES("alice")`, nil) + if err != nil { + t.Fatal("failed to insert record:", err) + } + + // Serialize the database to a file + tempFilename := TempFilename(t) + defer os.Remove(tempFilename) + if err := ioutil.WriteFile(tempFilename, sqlite3conn.Serialize(""), 0644); err != nil { + t.Fatalf("failed to write serialized database to disk") + } + + // Open the SQLite3 file, and test that contents are as expected. + db, err := sql.Open("sqlite3", tempFilename) + if err != nil { + t.Fatal("failed to open database:", err) + } + defer db.Close() + + rows, err := db.Query(`SELECT * FROM foo`) + if err != nil { + t.Fatal("failed to query database:", err) + } + defer rows.Close() + + rows.Next() + + var name string + rows.Scan(&name) + if exp, got := name, "alice"; exp != got { + t.Errorf("Expected %s for fetched result, but got %s:", exp, got) + } +} + +func TestDeserialize(t *testing.T) { + var sqlite3conn *SQLiteConn + d := SQLiteDriver{} + tempFilename := TempFilename(t) + defer os.Remove(tempFilename) + + // Create source database on disk. + conn, err := d.Open(tempFilename) + if err != nil { + t.Fatal("failed to open on-disk database:", err) + } + defer conn.Close() + sqlite3conn = conn.(*SQLiteConn) + _, err = sqlite3conn.Exec(`CREATE TABLE foo (name string)`, nil) + if err != nil { + t.Fatal("failed to create table:", err) + } + _, err = sqlite3conn.Exec(`INSERT INTO foo(name) VALUES("alice")`, nil) + if err != nil { + t.Fatal("failed to insert record:", err) + } + conn.Close() + + // Read database file bytes from disk. + b, err := ioutil.ReadFile(tempFilename) + if err != nil { + t.Fatal("failed to read database file on disk", err) + } + + // Deserialize file contents into memory. + conn, err = d.Open(":memory:") + if err != nil { + t.Fatal("failed to open in-memory database:", err) + } + sqlite3conn = conn.(*SQLiteConn) + defer conn.Close() + if err := sqlite3conn.Deserialize(b, ""); err != nil { + t.Fatal("failed to deserialize database", err) + } + + // Check database contents are as expected. + rows, err := sqlite3conn.Query(`SELECT * FROM foo`, nil) + if err != nil { + t.Fatal("failed to query database:", err) + } + if len(rows.Columns()) != 1 { + t.Fatal("incorrect number of columns returned:", len(rows.Columns())) + } + values := make([]driver.Value, 1) + rows.Next(values) + if v, ok := values[0].(string); !ok { + t.Fatalf("wrong type for value: %T", v) + } else if v != "alice" { + t.Fatal("wrong value returned", v) + } +} + func TestWAL(t *testing.T) { tempFilename := TempFilename(t) defer os.Remove(tempFilename) From 54e737335817aec0c2e5a2662140310c6e71a647 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Thu, 28 Jan 2021 21:51:50 -0500 Subject: [PATCH 02/39] Remove .github --- .github/FUNDING.yml | 8 --- .github/workflows/go.yaml | 120 -------------------------------------- 2 files changed, 128 deletions(-) delete mode 100644 .github/FUNDING.yml delete mode 100644 .github/workflows/go.yaml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml deleted file mode 100644 index 8f804949..00000000 --- a/.github/FUNDING.yml +++ /dev/null @@ -1,8 +0,0 @@ -# These are supported funding model platforms - -github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] -patreon: mattn # Replace with a single Patreon username -open_collective: mattn # Replace with a single Open Collective username -ko_fi: # Replace with a single Ko-fi username -tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel -custom: # Replace with a single custom sponsorship URL diff --git a/.github/workflows/go.yaml b/.github/workflows/go.yaml deleted file mode 100644 index 3ef6e384..00000000 --- a/.github/workflows/go.yaml +++ /dev/null @@ -1,120 +0,0 @@ -name: Go - -on: [push, pull_request] - -jobs: - - test: - name: Test - runs-on: ${{ matrix.os }} - defaults: - run: - shell: bash - - strategy: - matrix: - os: [ubuntu-latest, macos-latest] - go: ['1.12.17', '1.13.15', '1.14.7', '1.15'] - fail-fast: false - env: - OS: ${{ matrix.os }} - GO: ${{ matrix.go }} - steps: - - if: startsWith(matrix.os, 'macos') - run: brew update - - - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go }} - - - name: Get Build Tools - run: | - go get github.com/ory/go-acc - - - name: Add $GOPATH/bin to $PATH - run: | - echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH" - - - uses: actions/checkout@v2 - - - name: 'Tags: default' - run: go-acc . -- -race -v -tags "" - - - name: 'Tags: libsqlite3' - run: go-acc . -- -race -v -tags "libsqlite3" - - - name: 'Tags: full' - run: go-acc . -- -race -v -tags "sqlite_allow_uri_authority sqlite_app_armor sqlite_foreign_keys sqlite_fts5 sqlite_icu sqlite_introspect sqlite_json sqlite_preupdate_hook sqlite_secure_delete sqlite_see sqlite_stat4 sqlite_trace sqlite_userauth sqlite_vacuum_incr sqlite_vtable sqlite_unlock_notify" - - - name: 'Tags: vacuum' - run: go-acc . -- -race -v -tags "sqlite_vacuum_full" - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v1 - with: - env_vars: OS,GO - file: coverage.txt - - test-windows: - name: Test for Windows - runs-on: windows-latest - defaults: - run: - shell: bash - - strategy: - matrix: - go: ['1.12.17', '1.13.15', '1.14.7', '1.15'] - fail-fast: false - env: - OS: windows-latest - GO: ${{ matrix.go }} - steps: - - uses: msys2/setup-msys2@v2 - with: - update: true - install: mingw-w64-x86_64-toolchain mingw-w64-x86_64-sqlite3 - msystem: MINGW64 - path-type: inherit - - - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go }} - - - name: Get Build Tools - run: | - go get -u github.com/ory/go-acc - shell: msys2 {0} - - - name: Add $GOPATH/bin to $PATH - run: | - echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH" - shell: msys2 {0} - - - uses: actions/checkout@v2 - - - name: 'Tags: default' - run: go-acc . -- -race -v -tags "" - shell: msys2 {0} - - - name: 'Tags: libsqlite3' - run: go-acc . -- -race -v -tags "libsqlite3" - shell: msys2 {0} - - - name: 'Tags: full' - run: | - echo 'skip this test' - echo go-acc . -- -race -v -tags "sqlite_allow_uri_authority sqlite_app_armor sqlite_foreign_keys sqlite_fts5 sqlite_icu sqlite_introspect sqlite_json sqlite_preupdate_hook sqlite_secure_delete sqlite_see sqlite_stat4 sqlite_trace sqlite_userauth sqlite_vacuum_incr sqlite_vtable sqlite_unlock_notify" - shell: msys2 {0} - - - name: 'Tags: vacuum' - run: go-acc . -- -race -v -tags "sqlite_vacuum_full" - shell: msys2 {0} - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v1 - with: - env_vars: OS,GO - file: coverage.txt - -# based on: github.com/koron-go/_skeleton/.github/workflows/go.yml From e0abc440272c8e7aa023d6ed9eea6f6fab5a1918 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Thu, 28 Jan 2021 21:53:54 -0500 Subject: [PATCH 03/39] Add CircleCI testing support --- .circleci/config.yml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..443c312d --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,34 @@ + +# CircleCI 2.0 configuration file +# +# Check https://circleci.com/docs/2.0/language-go/ for more details +version: 2 +jobs: + test: + docker: + # specify the version + - image: circleci/golang:1.14 + + # Specify service dependencies here if necessary + # CircleCI maintains a library of pre-built images + # documented at https://circleci.com/docs/2.0/circleci-images/ + # - image: circleci/postgres:9.4 + + #### TEMPLATE_NOTE: go expects specific checkout path representing url + #### expecting it in the form of + #### /go/src/github.com/circleci/go-tool + #### /go/src/bitbucket.org/circleci/go-tool + working_directory: /go/src/github.com/rqlite/go-sqlite3 + steps: + - checkout + + # specify any bash command here prefixed with `run: ` + - run: go get -v -t -d ./... + - run: go vet + - run: go test -v ./... + +workflows: + version: 2 + build_and_test: + jobs: + - test From 75884b6f99a4780d1ed99ffa36146cb6da2f8a79 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Thu, 28 Jan 2021 21:59:13 -0500 Subject: [PATCH 04/39] Update README.md --- README.md | 590 +----------------------------------------------------- 1 file changed, 2 insertions(+), 588 deletions(-) diff --git a/README.md b/README.md index 746621f9..b5b3adff 100644 --- a/README.md +++ b/README.md @@ -1,592 +1,6 @@ go-sqlite3 ========== -[![GoDoc Reference](https://godoc.org/github.com/mattn/go-sqlite3?status.svg)](http://godoc.org/github.com/mattn/go-sqlite3) -[![GitHub Actions](https://github.com/mattn/go-sqlite3/workflows/Go/badge.svg)](https://github.com/mattn/go-sqlite3/actions?query=workflow%3AGo) -[![Financial Contributors on Open Collective](https://opencollective.com/mattn-go-sqlite3/all/badge.svg?label=financial+contributors)](https://opencollective.com/mattn-go-sqlite3) -[![codecov](https://codecov.io/gh/mattn/go-sqlite3/branch/master/graph/badge.svg)](https://codecov.io/gh/mattn/go-sqlite3) -[![Go Report Card](https://goreportcard.com/badge/github.com/mattn/go-sqlite3)](https://goreportcard.com/report/github.com/mattn/go-sqlite3) +[![Circle CI](https://circleci.com/gh/rqlite/go-sqlite3/tree/master.svg?style=svg)](https://circleci.com/gh/rqlite/go-sqlite3/tree/master) -Latest stable version is v1.14 or later not v2. - -~~**NOTE:** The increase to v2 was an accident. There were no major changes or features.~~ - -# Description - -sqlite3 driver conforming to the built-in database/sql interface - -Supported Golang version: See [.github/workflows/go.yaml](./.github/workflows/go.yaml) - -[This package follows the official Golang Release Policy.](https://golang.org/doc/devel/release.html#policy) - -### Overview - -- [go-sqlite3](#go-sqlite3) -- [Description](#description) - - [Overview](#overview) -- [Installation](#installation) -- [API Reference](#api-reference) -- [Connection String](#connection-string) - - [DSN Examples](#dsn-examples) -- [Features](#features) - - [Usage](#usage) - - [Feature / Extension List](#feature--extension-list) -- [Compilation](#compilation) - - [Android](#android) -- [ARM](#arm) -- [Cross Compile](#cross-compile) -- [Google Cloud Platform](#google-cloud-platform) - - [Linux](#linux) - - [Alpine](#alpine) - - [Fedora](#fedora) - - [Ubuntu](#ubuntu) - - [Mac OSX](#mac-osx) - - [Windows](#windows) - - [Errors](#errors) -- [User Authentication](#user-authentication) - - [Compile](#compile) - - [Usage](#usage-1) - - [Create protected database](#create-protected-database) - - [Password Encoding](#password-encoding) - - [Available Encoders](#available-encoders) - - [Restrictions](#restrictions) - - [Support](#support) - - [User Management](#user-management) - - [SQL](#sql) - - [Examples](#examples) - - [*SQLiteConn](#sqliteconn) - - [Attached database](#attached-database) -- [Extensions](#extensions) - - [Spatialite](#spatialite) -- [FAQ](#faq) -- [License](#license) -- [Author](#author) - -# Installation - -This package can be installed with the go get command: - - go get github.com/mattn/go-sqlite3 - -_go-sqlite3_ is *cgo* package. -If you want to build your app using go-sqlite3, you need gcc. -However, after you have built and installed _go-sqlite3_ with `go install github.com/mattn/go-sqlite3` (which requires gcc), you can build your app without relying on gcc in future. - -***Important: because this is a `CGO` enabled package you are required to set the environment variable `CGO_ENABLED=1` and have a `gcc` compile present within your path.*** - -# API Reference - -API documentation can be found here: http://godoc.org/github.com/mattn/go-sqlite3 - -Examples can be found under the [examples](./_example) directory - -# Connection String - -When creating a new SQLite database or connection to an existing one, with the file name additional options can be given. -This is also known as a DSN string. (Data Source Name). - -Options are append after the filename of the SQLite database. -The database filename and options are seperated by an `?` (Question Mark). -Options should be URL-encoded (see [url.QueryEscape](https://golang.org/pkg/net/url/#QueryEscape)). - -This also applies when using an in-memory database instead of a file. - -Options can be given using the following format: `KEYWORD=VALUE` and multiple options can be combined with the `&` ampersand. - -This library supports dsn options of SQLite itself and provides additional options. - -Boolean values can be one of: -* `0` `no` `false` `off` -* `1` `yes` `true` `on` - -| Name | Key | Value(s) | Description | -|------|-----|----------|-------------| -| UA - Create | `_auth` | - | Create User Authentication, for more information see [User Authentication](#user-authentication) | -| UA - Username | `_auth_user` | `string` | Username for User Authentication, for more information see [User Authentication](#user-authentication) | -| UA - Password | `_auth_pass` | `string` | Password for User Authentication, for more information see [User Authentication](#user-authentication) | -| UA - Crypt | `_auth_crypt` |
  • SHA1
  • SSHA1
  • SHA256
  • SSHA256
  • SHA384
  • SSHA384
  • SHA512
  • SSHA512
| Password encoder to use for User Authentication, for more information see [User Authentication](#user-authentication) | -| UA - Salt | `_auth_salt` | `string` | Salt to use if the configure password encoder requires a salt, for User Authentication, for more information see [User Authentication](#user-authentication) | -| Auto Vacuum | `_auto_vacuum` \| `_vacuum` |
  • `0` \| `none`
  • `1` \| `full`
  • `2` \| `incremental`
| For more information see [PRAGMA auto_vacuum](https://www.sqlite.org/pragma.html#pragma_auto_vacuum) | -| Busy Timeout | `_busy_timeout` \| `_timeout` | `int` | Specify value for sqlite3_busy_timeout. For more information see [PRAGMA busy_timeout](https://www.sqlite.org/pragma.html#pragma_busy_timeout) | -| Case Sensitive LIKE | `_case_sensitive_like` \| `_cslike` | `boolean` | For more information see [PRAGMA case_sensitive_like](https://www.sqlite.org/pragma.html#pragma_case_sensitive_like) | -| Defer Foreign Keys | `_defer_foreign_keys` \| `_defer_fk` | `boolean` | For more information see [PRAGMA defer_foreign_keys](https://www.sqlite.org/pragma.html#pragma_defer_foreign_keys) | -| Foreign Keys | `_foreign_keys` \| `_fk` | `boolean` | For more information see [PRAGMA foreign_keys](https://www.sqlite.org/pragma.html#pragma_foreign_keys) | -| Ignore CHECK Constraints | `_ignore_check_constraints` | `boolean` | For more information see [PRAGMA ignore_check_constraints](https://www.sqlite.org/pragma.html#pragma_ignore_check_constraints) | -| Immutable | `immutable` | `boolean` | For more information see [Immutable](https://www.sqlite.org/c3ref/open.html) | -| Journal Mode | `_journal_mode` \| `_journal` |
  • DELETE
  • TRUNCATE
  • PERSIST
  • MEMORY
  • WAL
  • OFF
| For more information see [PRAGMA journal_mode](https://www.sqlite.org/pragma.html#pragma_journal_mode) | -| Locking Mode | `_locking_mode` \| `_locking` |
  • NORMAL
  • EXCLUSIVE
| For more information see [PRAGMA locking_mode](https://www.sqlite.org/pragma.html#pragma_locking_mode) | -| Mode | `mode` |
  • ro
  • rw
  • rwc
  • memory
| Access Mode of the database. For more information see [SQLite Open](https://www.sqlite.org/c3ref/open.html) | -| Mutex Locking | `_mutex` |
  • no
  • full
| Specify mutex mode. | -| Query Only | `_query_only` | `boolean` | For more information see [PRAGMA query_only](https://www.sqlite.org/pragma.html#pragma_query_only) | -| Recursive Triggers | `_recursive_triggers` \| `_rt` | `boolean` | For more information see [PRAGMA recursive_triggers](https://www.sqlite.org/pragma.html#pragma_recursive_triggers) | -| Secure Delete | `_secure_delete` | `boolean` \| `FAST` | For more information see [PRAGMA secure_delete](https://www.sqlite.org/pragma.html#pragma_secure_delete) | -| Shared-Cache Mode | `cache` |
  • shared
  • private
| Set cache mode for more information see [sqlite.org](https://www.sqlite.org/sharedcache.html) | -| Synchronous | `_synchronous` \| `_sync` |
  • 0 \| OFF
  • 1 \| NORMAL
  • 2 \| FULL
  • 3 \| EXTRA
| For more information see [PRAGMA synchronous](https://www.sqlite.org/pragma.html#pragma_synchronous) | -| Time Zone Location | `_loc` | auto | Specify location of time format. | -| Transaction Lock | `_txlock` |
  • immediate
  • deferred
  • exclusive
| Specify locking behavior for transactions. | -| Writable Schema | `_writable_schema` | `Boolean` | When this pragma is on, the SQLITE_MASTER tables in which database can be changed using ordinary UPDATE, INSERT, and DELETE statements. Warning: misuse of this pragma can easily result in a corrupt database file. | -| Cache Size | `_cache_size` | `int` | Maximum cache size; default is 2000K (2M). See [PRAGMA cache_size](https://sqlite.org/pragma.html#pragma_cache_size) | - - -## DSN Examples - -``` -file:test.db?cache=shared&mode=memory -``` - -# Features - -This package allows additional configuration of features available within SQLite3 to be enabled or disabled by golang build constraints also known as build `tags`. - -[Click here for more information about build tags / constraints.](https://golang.org/pkg/go/build/#hdr-Build_Constraints) - -### Usage - -If you wish to build this library with additional extensions / features. -Use the following command. - -```bash -go build --tags "" -``` - -For available features see the extension list. -When using multiple build tags, all the different tags should be space delimted. - -Example: - -```bash -go build --tags "icu json1 fts5 secure_delete" -``` - -### Feature / Extension List - -| Extension | Build Tag | Description | -|-----------|-----------|-------------| -| Additional Statistics | sqlite_stat4 | This option adds additional logic to the ANALYZE command and to the query planner that can help SQLite to chose a better query plan under certain situations. The ANALYZE command is enhanced to collect histogram data from all columns of every index and store that data in the sqlite_stat4 table.

The query planner will then use the histogram data to help it make better index choices. The downside of this compile-time option is that it violates the query planner stability guarantee making it more difficult to ensure consistent performance in mass-produced applications.

SQLITE_ENABLE_STAT4 is an enhancement of SQLITE_ENABLE_STAT3. STAT3 only recorded histogram data for the left-most column of each index whereas the STAT4 enhancement records histogram data from all columns of each index.

The SQLITE_ENABLE_STAT3 compile-time option is a no-op and is ignored if the SQLITE_ENABLE_STAT4 compile-time option is used | -| Allow URI Authority | sqlite_allow_uri_authority | URI filenames normally throws an error if the authority section is not either empty or "localhost".

However, if SQLite is compiled with the SQLITE_ALLOW_URI_AUTHORITY compile-time option, then the URI is converted into a Uniform Naming Convention (UNC) filename and passed down to the underlying operating system that way | -| App Armor | sqlite_app_armor | When defined, this C-preprocessor macro activates extra code that attempts to detect misuse of the SQLite API, such as passing in NULL pointers to required parameters or using objects after they have been destroyed.

App Armor is not available under `Windows`. | -| Disable Load Extensions | sqlite_omit_load_extension | Loading of external extensions is enabled by default.

To disable extension loading add the build tag `sqlite_omit_load_extension`. | -| Foreign Keys | sqlite_foreign_keys | This macro determines whether enforcement of foreign key constraints is enabled or disabled by default for new database connections.

Each database connection can always turn enforcement of foreign key constraints on and off and run-time using the foreign_keys pragma.

Enforcement of foreign key constraints is normally off by default, but if this compile-time parameter is set to 1, enforcement of foreign key constraints will be on by default | -| Full Auto Vacuum | sqlite_vacuum_full | Set the default auto vacuum to full | -| Incremental Auto Vacuum | sqlite_vacuum_incr | Set the default auto vacuum to incremental | -| Full Text Search Engine | sqlite_fts5 | When this option is defined in the amalgamation, versions 5 of the full-text search engine (fts5) is added to the build automatically | -| International Components for Unicode | sqlite_icu | This option causes the International Components for Unicode or "ICU" extension to SQLite to be added to the build | -| Introspect PRAGMAS | sqlite_introspect | This option adds some extra PRAGMA statements.
  • PRAGMA function_list
  • PRAGMA module_list
  • PRAGMA pragma_list
| -| JSON SQL Functions | sqlite_json | When this option is defined in the amalgamation, the JSON SQL functions are added to the build automatically | -| Pre Update Hook | sqlite_preupdate_hook | Registers a callback function that is invoked prior to each INSERT, UPDATE, and DELETE operation on a database table. | -| Secure Delete | sqlite_secure_delete | This compile-time option changes the default setting of the secure_delete pragma.

When this option is not used, secure_delete defaults to off. When this option is present, secure_delete defaults to on.

The secure_delete setting causes deleted content to be overwritten with zeros. There is a small performance penalty since additional I/O must occur.

On the other hand, secure_delete can prevent fragments of sensitive information from lingering in unused parts of the database file after it has been deleted. See the documentation on the secure_delete pragma for additional information | -| Secure Delete (FAST) | sqlite_secure_delete_fast | For more information see [PRAGMA secure_delete](https://www.sqlite.org/pragma.html#pragma_secure_delete) | -| Tracing / Debug | sqlite_trace | Activate trace functions | -| User Authentication | sqlite_userauth | SQLite User Authentication see [User Authentication](#user-authentication) for more information. | - -# Compilation - -This package requires `CGO_ENABLED=1` ennvironment variable if not set by default, and the presence of the `gcc` compiler. - -If you need to add additional CFLAGS or LDFLAGS to the build command, and do not want to modify this package. Then this can be achieved by using the `CGO_CFLAGS` and `CGO_LDFLAGS` environment variables. - -## Android - -This package can be compiled for android. -Compile with: - -```bash -go build --tags "android" -``` - -For more information see [#201](https://github.com/mattn/go-sqlite3/issues/201) - -# ARM - -To compile for `ARM` use the following environment. - -```bash -env CC=arm-linux-gnueabihf-gcc CXX=arm-linux-gnueabihf-g++ \ - CGO_ENABLED=1 GOOS=linux GOARCH=arm GOARM=7 \ - go build -v -``` - -Additional information: -- [#242](https://github.com/mattn/go-sqlite3/issues/242) -- [#504](https://github.com/mattn/go-sqlite3/issues/504) - -# Cross Compile - -This library can be cross-compiled. - -In some cases you are required to the `CC` environment variable with the cross compiler. - -## Cross Compiling from MAC OSX -The simplest way to cross compile from OSX is to use [xgo](https://github.com/karalabe/xgo). - -Steps: -- Install [xgo](https://github.com/karalabe/xgo) (`go get github.com/karalabe/xgo`). -- Ensure that your project is within your `GOPATH`. -- Run `xgo local/path/to/project`. - -Please refer to the project's [README](https://github.com/karalabe/xgo/blob/master/README.md) for further information. - -# Google Cloud Platform - -Building on GCP is not possible because Google Cloud Platform does not allow `gcc` to be executed. - -Please work only with compiled final binaries. - -## Linux - -To compile this package on Linux you must install the development tools for your linux distribution. - -To compile under linux use the build tag `linux`. - -```bash -go build --tags "linux" -``` - -If you wish to link directly to libsqlite3 then you can use the `libsqlite3` build tag. - -``` -go build --tags "libsqlite3 linux" -``` - -### Alpine - -When building in an `alpine` container run the following command before building. - -``` -apk add --update gcc musl-dev -``` - -### Fedora - -```bash -sudo yum groupinstall "Development Tools" "Development Libraries" -``` - -### Ubuntu - -```bash -sudo apt-get install build-essential -``` - -## Mac OSX - -OSX should have all the tools present to compile this package, if not install XCode this will add all the developers tools. - -Required dependency - -```bash -brew install sqlite3 -``` - -For OSX there is an additional package install which is required if you wish to build the `icu` extension. - -This additional package can be installed with `homebrew`. - -```bash -brew upgrade icu4c -``` - -To compile for Mac OSX. - -```bash -go build --tags "darwin" -``` - -If you wish to link directly to libsqlite3 then you can use the `libsqlite3` build tag. - -``` -go build --tags "libsqlite3 darwin" -``` - -Additional information: -- [#206](https://github.com/mattn/go-sqlite3/issues/206) -- [#404](https://github.com/mattn/go-sqlite3/issues/404) - -## Windows - -To compile this package on Windows OS you must have the `gcc` compiler installed. - -1) Install a Windows `gcc` toolchain. -2) Add the `bin` folders to the Windows path if the installer did not do this by default. -3) Open a terminal for the TDM-GCC toolchain, can be found in the Windows Start menu. -4) Navigate to your project folder and run the `go build ...` command for this package. - -For example the TDM-GCC Toolchain can be found [here](https://sourceforge.net/projects/tdm-gcc/). - -## Errors - -- Compile error: `can not be used when making a shared object; recompile with -fPIC` - - When receiving a compile time error referencing recompile with `-FPIC` then you - are probably using a hardend system. - - You can compile the library on a hardend system with the following command. - - ```bash - go build -ldflags '-extldflags=-fno-PIC' - ``` - - More details see [#120](https://github.com/mattn/go-sqlite3/issues/120) - -- Can't build go-sqlite3 on windows 64bit. - - > Probably, you are using go 1.0, go1.0 has a problem when it comes to compiling/linking on windows 64bit. - > See: [#27](https://github.com/mattn/go-sqlite3/issues/27) - -- `go get github.com/mattn/go-sqlite3` throws compilation error. - - `gcc` throws: `internal compiler error` - - Remove the download repository from your disk and try re-install with: - - ```bash - go install github.com/mattn/go-sqlite3 - ``` - -# User Authentication - -This package supports the SQLite User Authentication module. - -## Compile - -To use the User authentication module the package has to be compiled with the tag `sqlite_userauth`. See [Features](#features). - -## Usage - -### Create protected database - -To create a database protected by user authentication provide the following argument to the connection string `_auth`. -This will enable user authentication within the database. This option however requires two additional arguments: - -- `_auth_user` -- `_auth_pass` - -When `_auth` is present on the connection string user authentication will be enabled and the provided user will be created -as an `admin` user. After initial creation, the parameter `_auth` has no effect anymore and can be omitted from the connection string. - -Example connection string: - -Create an user authentication database with user `admin` and password `admin`. - -`file:test.s3db?_auth&_auth_user=admin&_auth_pass=admin` - -Create an user authentication database with user `admin` and password `admin` and use `SHA1` for the password encoding. - -`file:test.s3db?_auth&_auth_user=admin&_auth_pass=admin&_auth_crypt=sha1` - -### Password Encoding - -The passwords within the user authentication module of SQLite are encoded with the SQLite function `sqlite_cryp`. -This function uses a ceasar-cypher which is quite insecure. -This library provides several additional password encoders which can be configured through the connection string. - -The password cypher can be configured with the key `_auth_crypt`. And if the configured password encoder also requires an -salt this can be configured with `_auth_salt`. - -#### Available Encoders - -- SHA1 -- SSHA1 (Salted SHA1) -- SHA256 -- SSHA256 (salted SHA256) -- SHA384 -- SSHA384 (salted SHA384) -- SHA512 -- SSHA512 (salted SHA512) - -### Restrictions - -Operations on the database regarding to user management can only be preformed by an administrator user. - -### Support - -The user authentication supports two kinds of users - -- administrators -- regular users - -### User Management - -User management can be done by directly using the `*SQLiteConn` or by SQL. - -#### SQL - -The following sql functions are available for user management. - -| Function | Arguments | Description | -|----------|-----------|-------------| -| `authenticate` | username `string`, password `string` | Will authenticate an user, this is done by the connection; and should not be used manually. | -| `auth_user_add` | username `string`, password `string`, admin `int` | This function will add an user to the database.
if the database is not protected by user authentication it will enable it. Argument `admin` is an integer identifying if the added user should be an administrator. Only Administrators can add administrators. | -| `auth_user_change` | username `string`, password `string`, admin `int` | Function to modify an user. Users can change their own password, but only an administrator can change the administrator flag. | -| `authUserDelete` | username `string` | Delete an user from the database. Can only be used by an administrator. The current logged in administrator cannot be deleted. This is to make sure their is always an administrator remaining. | - -These functions will return an integer. - -- 0 (SQLITE_OK) -- 23 (SQLITE_AUTH) Failed to perform due to authentication or insufficient privileges - -##### Examples - -```sql -// Autheticate user -// Create Admin User -SELECT auth_user_add('admin2', 'admin2', 1); - -// Change password for user -SELECT auth_user_change('user', 'userpassword', 0); - -// Delete user -SELECT user_delete('user'); -``` - -#### *SQLiteConn - -The following functions are available for User authentication from the `*SQLiteConn`. - -| Function | Description | -|----------|-------------| -| `Authenticate(username, password string) error` | Authenticate user | -| `AuthUserAdd(username, password string, admin bool) error` | Add user | -| `AuthUserChange(username, password string, admin bool) error` | Modify user | -| `AuthUserDelete(username string) error` | Delete user | - -### Attached database - -When using attached databases. SQLite will use the authentication from the `main` database for the attached database(s). - -# Extensions - -If you want your own extension to be listed here or you want to add a reference to an extension; please submit an Issue for this. - -## Spatialite - -Spatialite is available as an extension to SQLite, and can be used in combination with this repository. -For an example see [shaxbee/go-spatialite](https://github.com/shaxbee/go-spatialite). - -## extension-functions.c from SQLite3 Contrib - -extension-functions.c is available as an extension to SQLite, and provides the following functions: - -- Math: acos, asin, atan, atn2, atan2, acosh, asinh, atanh, difference, degrees, radians, cos, sin, tan, cot, cosh, sinh, tanh, coth, exp, log, log10, power, sign, sqrt, square, ceil, floor, pi. -- String: replicate, charindex, leftstr, rightstr, ltrim, rtrim, trim, replace, reverse, proper, padl, padr, padc, strfilter. -- Aggregate: stdev, variance, mode, median, lower_quartile, upper_quartile - -For an example see [dinedal/go-sqlite3-extension-functions](https://github.com/dinedal/go-sqlite3-extension-functions). - -# FAQ - -- Getting insert error while query is opened. - - > You can pass some arguments into the connection string, for example, a URI. - > See: [#39](https://github.com/mattn/go-sqlite3/issues/39) - -- Do you want to cross compile? mingw on Linux or Mac? - - > See: [#106](https://github.com/mattn/go-sqlite3/issues/106) - > See also: http://www.limitlessfx.com/cross-compile-golang-app-for-windows-from-linux.html - -- Want to get time.Time with current locale - - Use `_loc=auto` in SQLite3 filename schema like `file:foo.db?_loc=auto`. - -- Can I use this in multiple routines concurrently? - - Yes for readonly. But, No for writable. See [#50](https://github.com/mattn/go-sqlite3/issues/50), [#51](https://github.com/mattn/go-sqlite3/issues/51), [#209](https://github.com/mattn/go-sqlite3/issues/209), [#274](https://github.com/mattn/go-sqlite3/issues/274). - -- Why I'm getting `no such table` error? - - Why is it racy if I use a `sql.Open("sqlite3", ":memory:")` database? - - Each connection to `":memory:"` opens a brand new in-memory sql database, so if - the stdlib's sql engine happens to open another connection and you've only - specified `":memory:"`, that connection will see a brand new database. A - workaround is to use `"file::memory:?cache=shared"` (or `"file:foobar?mode=memory&cache=shared"`). Every - connection to this string will point to the same in-memory database. - - Note that if the last database connection in the pool closes, the in-memory database is deleted. Make sure the [max idle connection limit](https://golang.org/pkg/database/sql/#DB.SetMaxIdleConns) is > 0, and the [connection lifetime](https://golang.org/pkg/database/sql/#DB.SetConnMaxLifetime) is infinite. - - For more information see - * [#204](https://github.com/mattn/go-sqlite3/issues/204) - * [#511](https://github.com/mattn/go-sqlite3/issues/511) - * https://www.sqlite.org/sharedcache.html#shared_cache_and_in_memory_databases - * https://www.sqlite.org/inmemorydb.html#sharedmemdb - -- Reading from database with large amount of goroutines fails on OSX. - - OS X limits OS-wide to not have more than 1000 files open simultaneously by default. - - For more information see [#289](https://github.com/mattn/go-sqlite3/issues/289) - -- Trying to execute a `.` (dot) command throws an error. - - Error: `Error: near ".": syntax error` - Dot command are part of SQLite3 CLI not of this library. - - You need to implement the feature or call the sqlite3 cli. - - More information see [#305](https://github.com/mattn/go-sqlite3/issues/305) - -- Error: `database is locked` - - When you get a database is locked. Please use the following options. - - Add to DSN: `cache=shared` - - Example: - ```go - db, err := sql.Open("sqlite3", "file:locked.sqlite?cache=shared") - ``` - - Second please set the database connections of the SQL package to 1. - - ```go - db.SetMaxOpenConns(1) - ``` - - More information see [#209](https://github.com/mattn/go-sqlite3/issues/209) - -## Contributors - -### Code Contributors - -This project exists thanks to all the people who contribute. [[Contribute](CONTRIBUTING.md)]. - - -### Financial Contributors - -Become a financial contributor and help us sustain our community. [[Contribute](https://opencollective.com/mattn-go-sqlite3/contribute)] - -#### Individuals - - - -#### Organizations - -Support this project with your organization. Your logo will show up here with a link to your website. [[Contribute](https://opencollective.com/mattn-go-sqlite3/contribute)] - - - - - - - - - - - - -# License - -MIT: http://mattn.mit-license.org/2018 - -sqlite3-binding.c, sqlite3-binding.h, sqlite3ext.h - -The -binding suffix was added to avoid build failures under gccgo. - -In this repository, those files are an amalgamation of code that was copied from SQLite3. The license of that code is the same as the license of SQLite3. - -# Author - -Yasuhiro Matsumoto (a.k.a mattn) - -G.J.R. Timmer +See upstream for README. From eeaa9571459ac1e29148ccb47018cbf540afaf97 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Thu, 28 Jan 2021 22:53:52 -0500 Subject: [PATCH 05/39] Update go.mod --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 3d0854a0..847872a1 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ -module github.com/mattn/go-sqlite3 +module github.com/rqlite/go-sqlite3 go 1.12 From 645c9cbedaa190fc109e751ad6be69d580d54a92 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Sun, 31 Jan 2021 14:13:44 -0500 Subject: [PATCH 06/39] Provide SQLite with its own memory for deserialize https://github.com/rqlite/rqlite/pull/739 --- sqlite3.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/sqlite3.go b/sqlite3.go index 61a381ae..0dadc146 100644 --- a/sqlite3.go +++ b/sqlite3.go @@ -931,6 +931,10 @@ func (c *SQLiteConn) Serialize(schema string) []byte { // byte slice. If deserelization fails, error will contain the return code // of the underlying SQLite API call. // +// Because the byte slice is in Go-controlled memory, this function +// makes a copy of the data in C-controlled memory, before passing the +// data to the SQLite library. +// // See https://www.sqlite.org/c3ref/deserialize.html func (c *SQLiteConn) Deserialize(b []byte, schema string) error { if schema == "" { @@ -941,8 +945,9 @@ func (c *SQLiteConn) Deserialize(b []byte, schema string) error { defer C.free(unsafe.Pointer(zSchema)) rc := C.sqlite3_deserialize(c.db, zSchema, - (*C.uint8_t)(unsafe.Pointer(&b[0])), - C.sqlite3_int64(len(b)), C.sqlite3_int64(len(b)), 0) + (*C.uint8_t)(C.CBytes(b)), + C.sqlite3_int64(len(b)), C.sqlite3_int64(len(b)), + C.SQLITE_DESERIALIZE_FREEONCLOSE|C.SQLITE_DESERIALIZE_RESIZEABLE) if rc != 0 { return fmt.Errorf("deserialize failed with return %v", rc) } From 274a6085af46188137283213158969b7dd6dabdb Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Sun, 31 Jan 2021 15:49:14 -0500 Subject: [PATCH 07/39] Revert "Provide SQLite with its own memory for deserialize" This reverts commit 645c9cbedaa190fc109e751ad6be69d580d54a92. --- sqlite3.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/sqlite3.go b/sqlite3.go index 0dadc146..61a381ae 100644 --- a/sqlite3.go +++ b/sqlite3.go @@ -931,10 +931,6 @@ func (c *SQLiteConn) Serialize(schema string) []byte { // byte slice. If deserelization fails, error will contain the return code // of the underlying SQLite API call. // -// Because the byte slice is in Go-controlled memory, this function -// makes a copy of the data in C-controlled memory, before passing the -// data to the SQLite library. -// // See https://www.sqlite.org/c3ref/deserialize.html func (c *SQLiteConn) Deserialize(b []byte, schema string) error { if schema == "" { @@ -945,9 +941,8 @@ func (c *SQLiteConn) Deserialize(b []byte, schema string) error { defer C.free(unsafe.Pointer(zSchema)) rc := C.sqlite3_deserialize(c.db, zSchema, - (*C.uint8_t)(C.CBytes(b)), - C.sqlite3_int64(len(b)), C.sqlite3_int64(len(b)), - C.SQLITE_DESERIALIZE_FREEONCLOSE|C.SQLITE_DESERIALIZE_RESIZEABLE) + (*C.uint8_t)(unsafe.Pointer(&b[0])), + C.sqlite3_int64(len(b)), C.sqlite3_int64(len(b)), 0) if rc != 0 { return fmt.Errorf("deserialize failed with return %v", rc) } From 16c671d34824dfb5e3e8c34123796bc2be030fd8 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Sun, 31 Jan 2021 16:05:20 -0500 Subject: [PATCH 08/39] Better comment for Deserialize --- sqlite3.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sqlite3.go b/sqlite3.go index 61a381ae..1bdd6cee 100644 --- a/sqlite3.go +++ b/sqlite3.go @@ -931,6 +931,10 @@ func (c *SQLiteConn) Serialize(schema string) []byte { // byte slice. If deserelization fails, error will contain the return code // of the underlying SQLite API call. // +// When this function returns, the connection is referencing database +// data in Go space, so the connection and associated database must be copied +// immediately if it is to be used further. +// // See https://www.sqlite.org/c3ref/deserialize.html func (c *SQLiteConn) Deserialize(b []byte, schema string) error { if schema == "" { From 369e217ae9bf45d56a2f42b9a9b7b48e96738067 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Mon, 8 Feb 2021 08:58:01 -0500 Subject: [PATCH 09/39] Free memory allocated by sqlite due to serialize --- sqlite3.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sqlite3.go b/sqlite3.go index 1bdd6cee..9c63b55c 100644 --- a/sqlite3.go +++ b/sqlite3.go @@ -923,6 +923,7 @@ func (c *SQLiteConn) Serialize(schema string) []byte { if ptr == nil { return nil } + defer C.sqlite3_free(unsafe.Pointer(ptr)) return C.GoBytes(unsafe.Pointer(ptr), C.int(sz)) } From eb446c9c9f9ee5b241a9f7ecceb0402a7e39f200 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Sat, 31 Jul 2021 09:58:32 -0400 Subject: [PATCH 10/39] Return non-nil result https://github.com/mattn/go-sqlite3/issues/963 --- sqlite3.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sqlite3.go b/sqlite3.go index 36a1daba..7a10a8a6 100644 --- a/sqlite3.go +++ b/sqlite3.go @@ -829,6 +829,10 @@ func (c *SQLiteConn) exec(ctx context.Context, query string, args []namedValue) tail := s.(*SQLiteStmt).t s.Close() if tail == "" { + if res == nil { + // https://github.com/mattn/go-sqlite3/issues/963 + res = &SQLiteResult{0, 0} + } return res, nil } query = tail From da26a81d325d643527fa4f5c0577bdf840666bbc Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Thu, 5 Aug 2021 20:10:14 -0400 Subject: [PATCH 11/39] Enable some new SQLite compile options --- sqlite3.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sqlite3.go b/sqlite3.go index 36a1daba..4aa1ac7e 100644 --- a/sqlite3.go +++ b/sqlite3.go @@ -15,11 +15,17 @@ package sqlite3 #cgo CFLAGS: -DHAVE_USLEEP=1 #cgo CFLAGS: -DSQLITE_ENABLE_FTS3 #cgo CFLAGS: -DSQLITE_ENABLE_FTS3_PARENTHESIS +#cgo CFLAGS: -DSQLITE_ENABLE_FTS3_TOKENIZER #cgo CFLAGS: -DSQLITE_TRACE_SIZE_LIMIT=15 #cgo CFLAGS: -DSQLITE_OMIT_DEPRECATED #cgo CFLAGS: -DSQLITE_ENABLE_DESERIALIZE #cgo CFLAGS: -DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1 #cgo CFLAGS: -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT +#cgo CFLAGS: -DSQLITE_OMIT_SHARED_CACHE +#cgo CFLAGS: -DENABLE_JSON1 +#cgo CFLAGS: -DSQLITE_ENABLE_FTS4 +#cgo CFLAGS: -DSQLITE_ENABLE_FTS5 +#cgo CFLAGS: -DSQLITE_ENABLE_DBSTAT_VTAB #cgo CFLAGS: -Wno-deprecated-declarations #cgo linux,!android CFLAGS: -DHAVE_PREAD64=1 -DHAVE_PWRITE64=1 #ifndef USE_LIBSQLITE3 From 41c9bb68d7fd3f531940911c9ad34a94493c4ae0 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Thu, 5 Aug 2021 20:11:36 -0400 Subject: [PATCH 12/39] Correct macro for JSON1 --- sqlite3.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite3.go b/sqlite3.go index 4aa1ac7e..9b39defe 100644 --- a/sqlite3.go +++ b/sqlite3.go @@ -22,7 +22,7 @@ package sqlite3 #cgo CFLAGS: -DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1 #cgo CFLAGS: -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT #cgo CFLAGS: -DSQLITE_OMIT_SHARED_CACHE -#cgo CFLAGS: -DENABLE_JSON1 +#cgo CFLAGS: -DSQLITE_ENABLE_JSON1 #cgo CFLAGS: -DSQLITE_ENABLE_FTS4 #cgo CFLAGS: -DSQLITE_ENABLE_FTS5 #cgo CFLAGS: -DSQLITE_ENABLE_DBSTAT_VTAB From 2a971cdbdfdaa46c06e5234eaf1e83b16dc17b25 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Thu, 5 Aug 2021 20:16:16 -0400 Subject: [PATCH 13/39] Remove some of the FTS macros --- sqlite3.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/sqlite3.go b/sqlite3.go index 9b39defe..3d11f44e 100644 --- a/sqlite3.go +++ b/sqlite3.go @@ -15,7 +15,6 @@ package sqlite3 #cgo CFLAGS: -DHAVE_USLEEP=1 #cgo CFLAGS: -DSQLITE_ENABLE_FTS3 #cgo CFLAGS: -DSQLITE_ENABLE_FTS3_PARENTHESIS -#cgo CFLAGS: -DSQLITE_ENABLE_FTS3_TOKENIZER #cgo CFLAGS: -DSQLITE_TRACE_SIZE_LIMIT=15 #cgo CFLAGS: -DSQLITE_OMIT_DEPRECATED #cgo CFLAGS: -DSQLITE_ENABLE_DESERIALIZE @@ -23,8 +22,6 @@ package sqlite3 #cgo CFLAGS: -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT #cgo CFLAGS: -DSQLITE_OMIT_SHARED_CACHE #cgo CFLAGS: -DSQLITE_ENABLE_JSON1 -#cgo CFLAGS: -DSQLITE_ENABLE_FTS4 -#cgo CFLAGS: -DSQLITE_ENABLE_FTS5 #cgo CFLAGS: -DSQLITE_ENABLE_DBSTAT_VTAB #cgo CFLAGS: -Wno-deprecated-declarations #cgo linux,!android CFLAGS: -DHAVE_PREAD64=1 -DHAVE_PWRITE64=1 From 27f06c421992669daeac22379baf80c376c5b2c0 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Fri, 20 Aug 2021 09:23:32 -0400 Subject: [PATCH 14/39] Remove DSQLITE_ENABLE_DESERIALIZE No longer needed See https://www.sqlite.org/releaselog/3_36_0.html --- sqlite3.go | 1 - 1 file changed, 1 deletion(-) diff --git a/sqlite3.go b/sqlite3.go index 40f9b6f2..074a4655 100644 --- a/sqlite3.go +++ b/sqlite3.go @@ -17,7 +17,6 @@ package sqlite3 #cgo CFLAGS: -DSQLITE_ENABLE_FTS3_PARENTHESIS #cgo CFLAGS: -DSQLITE_TRACE_SIZE_LIMIT=15 #cgo CFLAGS: -DSQLITE_OMIT_DEPRECATED -#cgo CFLAGS: -DSQLITE_ENABLE_DESERIALIZE #cgo CFLAGS: -DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1 #cgo CFLAGS: -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT #cgo CFLAGS: -DSQLITE_OMIT_SHARED_CACHE From 042f223e308b9be5855abdfd8cb1c9390851f64c Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Wed, 1 Sep 2021 08:04:16 -0400 Subject: [PATCH 15/39] Increase default max memory size For now, increase to the max supported by sqlite3_malloc64(), as per https://sqlite.org/forum/forumpost/b1ba232667130afa?t=h. --- sqlite3.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sqlite3.go b/sqlite3.go index 40f9b6f2..c247ddb5 100644 --- a/sqlite3.go +++ b/sqlite3.go @@ -23,6 +23,7 @@ package sqlite3 #cgo CFLAGS: -DSQLITE_OMIT_SHARED_CACHE #cgo CFLAGS: -DSQLITE_ENABLE_JSON1 #cgo CFLAGS: -DSQLITE_ENABLE_DBSTAT_VTAB +#cgo CFLAGS: -DSQLITE_MEMDB_DEFAULT_MAXSIZE=2147483648 #cgo CFLAGS: -Wno-deprecated-declarations #cgo linux,!android CFLAGS: -DHAVE_PREAD64=1 -DHAVE_PWRITE64=1 #ifndef USE_LIBSQLITE3 From 788d503191ea8b4a449f9385a46d8e881edb9e79 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Tue, 17 May 2022 20:22:57 -0400 Subject: [PATCH 16/39] Merge upstream --- sqlite3-binding.c | 477 ++++++++++++++++++++++++---------- sqlite3-binding.h | 16 +- sqlite3_opt_json1.go | 13 - sqlite3_opt_preupdate_hook.go | 2 +- sqlite3_opt_preupdate_omit.go | 2 +- 5 files changed, 356 insertions(+), 154 deletions(-) delete mode 100644 sqlite3_opt_json1.go diff --git a/sqlite3-binding.c b/sqlite3-binding.c index d079f1ad..102821bb 100644 --- a/sqlite3-binding.c +++ b/sqlite3-binding.c @@ -1,7 +1,7 @@ #ifndef USE_LIBSQLITE3 /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.38.0. By combining all the individual C code files into this +** version 3.38.5. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -453,9 +453,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.38.0" -#define SQLITE_VERSION_NUMBER 3038000 -#define SQLITE_SOURCE_ID "2022-02-22 18:58:40 40fa792d359f84c3b9e9d6623743e1a59826274e221df1bde8f47086968a1bab" +#define SQLITE_VERSION "3.38.5" +#define SQLITE_VERSION_NUMBER 3038005 +#define SQLITE_SOURCE_ID "2022-05-06 15:25:27 78d9c993d404cdfaa7fdd2973fa1052e3da9f66215cff9c5540ebe55c407d9fe" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -5286,6 +5286,10 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** even empty strings, are always zero-terminated. ^The return ** value from sqlite3_column_blob() for a zero-length BLOB is a NULL pointer. ** +** ^Strings returned by sqlite3_column_text16() always have the endianness +** which is native to the platform, regardless of the text encoding set +** for the database. +** ** Warning: ^The object returned by [sqlite3_column_value()] is an ** [unprotected sqlite3_value] object. In a multithreaded environment, ** an unprotected sqlite3_value object may only be used safely with @@ -5299,7 +5303,7 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** [application-defined SQL functions] or [virtual tables], not within ** top-level application code. ** -** The these routines may attempt to convert the datatype of the result. +** These routines may attempt to convert the datatype of the result. ** ^For example, if the internal representation is FLOAT and a text result ** is requested, [sqlite3_snprintf()] is used internally to perform the ** conversion automatically. ^(The following table details the conversions @@ -5324,7 +5328,7 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** TEXT BLOB No change ** BLOB INTEGER [CAST] to INTEGER ** BLOB FLOAT [CAST] to REAL -** BLOB TEXT Add a zero terminator if needed +** BLOB TEXT [CAST] to TEXT, ensure zero terminator ** ** )^ ** @@ -10074,7 +10078,7 @@ SQLITE_API int sqlite3_vtab_in_next(sqlite3_value *pVal, sqlite3_value **ppOut); ** ^When xBestIndex returns, the sqlite3_value object returned by ** sqlite3_vtab_rhs_value() is automatically deallocated. ** -** The "_rhs_" in the name of this routine is an appreviation for +** The "_rhs_" in the name of this routine is an abbreviation for ** "Right-Hand Side". */ SQLITE_API int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value **ppVal); @@ -19926,6 +19930,7 @@ SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr*); SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr*, u8); SQLITE_PRIVATE int sqlite3ExprIsConstantOrGroupBy(Parse*, Expr*, ExprList*); SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr*,int); +SQLITE_PRIVATE int sqlite3ExprIsTableConstraint(Expr*,const SrcItem*); #ifdef SQLITE_ENABLE_CURSOR_HINTS SQLITE_PRIVATE int sqlite3ExprContainsSubquery(Expr*); #endif @@ -20365,7 +20370,12 @@ SQLITE_PRIVATE int sqlite3VtabCallCreate(sqlite3*, int, const char *, char **); SQLITE_PRIVATE int sqlite3VtabCallConnect(Parse*, Table*); SQLITE_PRIVATE int sqlite3VtabCallDestroy(sqlite3*, int, const char *); SQLITE_PRIVATE int sqlite3VtabBegin(sqlite3 *, VTable *); + SQLITE_PRIVATE FuncDef *sqlite3VtabOverloadFunction(sqlite3 *,FuncDef*, int nArg, Expr*); +#if (defined(SQLITE_ENABLE_DBPAGE_VTAB) || defined(SQLITE_TEST)) \ + && !defined(SQLITE_OMIT_VIRTUALTABLE) +SQLITE_PRIVATE void sqlite3VtabWriteAll(sqlite3_index_info*); +#endif SQLITE_PRIVATE sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context*); SQLITE_PRIVATE int sqlite3VdbeParameterIndex(Vdbe*, const char*, int); SQLITE_PRIVATE int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *); @@ -23655,7 +23665,7 @@ static int toLocaltime( p->D = sLocal.tm_mday; p->h = sLocal.tm_hour; p->m = sLocal.tm_min; - p->s = sLocal.tm_sec; + p->s = sLocal.tm_sec + (p->iJD%1000)*0.001; p->validYMD = 1; p->validHMS = 1; p->validJD = 0; @@ -29329,8 +29339,9 @@ SQLITE_PRIVATE char *sqlite3DbSpanDup(sqlite3 *db, const char *zStart, const cha ** Free any prior content in *pz and replace it with a copy of zNew. */ SQLITE_PRIVATE void sqlite3SetString(char **pz, sqlite3 *db, const char *zNew){ + char *z = sqlite3DbStrDup(db, zNew); sqlite3DbFree(db, *pz); - *pz = sqlite3DbStrDup(db, zNew); + *pz = z; } /* @@ -39683,11 +39694,17 @@ static int unixShmLock( int flags /* What to do with the lock */ ){ unixFile *pDbFd = (unixFile*)fd; /* Connection holding shared memory */ - unixShm *p = pDbFd->pShm; /* The shared memory being locked */ - unixShmNode *pShmNode = p->pShmNode; /* The underlying file iNode */ + unixShm *p; /* The shared memory being locked */ + unixShmNode *pShmNode; /* The underlying file iNode */ int rc = SQLITE_OK; /* Result code */ u16 mask; /* Mask of locks to take or release */ - int *aLock = pShmNode->aLock; + int *aLock; + + p = pDbFd->pShm; + if( p==0 ) return SQLITE_IOERR_SHMLOCK; + pShmNode = p->pShmNode; + if( NEVER(pShmNode==0) ) return SQLITE_IOERR_SHMLOCK; + aLock = pShmNode->aLock; assert( pShmNode==pDbFd->pInode->pShmNode ); assert( pShmNode->pInode==pDbFd->pInode ); @@ -46975,10 +46992,14 @@ static int winShmLock( winFile *pDbFd = (winFile*)fd; /* Connection holding shared memory */ winShm *p = pDbFd->pShm; /* The shared memory being locked */ winShm *pX; /* For looping over all siblings */ - winShmNode *pShmNode = p->pShmNode; + winShmNode *pShmNode; int rc = SQLITE_OK; /* Result code */ u16 mask; /* Mask of locks to take or release */ + if( p==0 ) return SQLITE_IOERR_SHMLOCK; + pShmNode = p->pShmNode; + if( NEVER(pShmNode==0) ) return SQLITE_IOERR_SHMLOCK; + assert( ofst>=0 && ofst+n<=SQLITE_SHM_NLOCK ); assert( n>=1 ); assert( flags==(SQLITE_SHM_LOCK | SQLITE_SHM_SHARED) @@ -50908,7 +50929,8 @@ SQLITE_PRIVATE void sqlite3PcacheDrop(PgHdr *p){ ** make it so. */ SQLITE_PRIVATE void sqlite3PcacheMakeDirty(PgHdr *p){ - assert( p->nRef>0 ); + assert( p->nRef>0 || p->pCache->bPurgeable==0 ); + testcase( p->nRef==0 ); assert( sqlite3PcachePageSanity(p) ); if( p->flags & (PGHDR_CLEAN|PGHDR_DONT_WRITE) ){ /*OPTIMIZATION-IF-FALSE*/ p->flags &= ~PGHDR_DONT_WRITE; @@ -56040,6 +56062,9 @@ static int pager_playback(Pager *pPager, int isHot){ goto end_playback; } pPager->dbSize = mxPg; + if( pPager->mxPgnomxPgno = mxPg; + } } /* Copy original pages out of the journal and back into the @@ -65399,7 +65424,9 @@ struct MemPage { u8 *apOvfl[4]; /* Pointers to the body of overflow cells */ BtShared *pBt; /* Pointer to BtShared that this page is part of */ u8 *aData; /* Pointer to disk image of the page data */ - u8 *aDataEnd; /* One byte past the end of usable data */ + u8 *aDataEnd; /* One byte past the end of the entire page - not just + ** the usable space, the entire page. Used to prevent + ** corruption-induced of buffer overflow. */ u8 *aCellIdx; /* The cell index area */ u8 *aDataOfst; /* Same as aData for leaves. aData+4 for interior */ DbPage *pDbPage; /* Pager page handle */ @@ -67740,6 +67767,8 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ ** fragmented bytes within the page. */ memcpy(&aData[iAddr], &aData[pc], 2); aData[hdr+7] += (u8)x; + testcase( pc+x>maxPC ); + return &aData[pc]; }else if( x+pc > maxPC ){ /* This slot extends off the end of the usable part of the page */ *pRc = SQLITE_CORRUPT_PAGE(pPg); @@ -68184,7 +68213,7 @@ static int btreeInitPage(MemPage *pPage){ pPage->nOverflow = 0; pPage->cellOffset = pPage->hdrOffset + 8 + pPage->childPtrSize; pPage->aCellIdx = data + pPage->childPtrSize + 8; - pPage->aDataEnd = pPage->aData + pBt->usableSize; + pPage->aDataEnd = pPage->aData + pBt->pageSize; pPage->aDataOfst = pPage->aData + pPage->childPtrSize; /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the ** number of cells on the page. */ @@ -68219,7 +68248,7 @@ static void zeroPage(MemPage *pPage, int flags){ u8 hdr = pPage->hdrOffset; u16 first; - assert( sqlite3PagerPagenumber(pPage->pDbPage)==pPage->pgno ); + assert( sqlite3PagerPagenumber(pPage->pDbPage)==pPage->pgno || CORRUPT_DB ); assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage ); assert( sqlite3PagerGetData(pPage->pDbPage) == data ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); @@ -68235,7 +68264,7 @@ static void zeroPage(MemPage *pPage, int flags){ pPage->nFree = (u16)(pBt->usableSize - first); decodeFlags(pPage, flags); pPage->cellOffset = first; - pPage->aDataEnd = &data[pBt->usableSize]; + pPage->aDataEnd = &data[pBt->pageSize]; pPage->aCellIdx = &data[first]; pPage->aDataOfst = &data[pPage->childPtrSize]; pPage->nOverflow = 0; @@ -68361,7 +68390,7 @@ static int getAndInitPage( goto getAndInitPage_error2; } } - assert( (*ppPage)->pgno==pgno ); + assert( (*ppPage)->pgno==pgno || CORRUPT_DB ); assert( (*ppPage)->aData==sqlite3PagerGetData(pDbPage) ); /* If obtaining a child page for a cursor, we must verify that the page is @@ -71448,7 +71477,7 @@ static int moveToRoot(BtCursor *pCur){ pCur->curIntKey = pCur->pPage->intKey; } pRoot = pCur->pPage; - assert( pRoot->pgno==pCur->pgnoRoot ); + assert( pRoot->pgno==pCur->pgnoRoot || CORRUPT_DB ); /* If pCur->pKeyInfo is not NULL, then the caller that opened this cursor ** expected to open it on an index b-tree. Otherwise, if pKeyInfo is @@ -71939,7 +71968,7 @@ SQLITE_PRIVATE int sqlite3BtreeIndexMoveto( assert( lwr==upr+1 || (pPage->intKey && !pPage->leaf) ); assert( pPage->isInit ); if( pPage->leaf ){ - assert( pCur->ixpPage->nCell ); + assert( pCur->ixpPage->nCell || CORRUPT_DB ); pCur->ix = (u16)idx; *pRes = c; rc = SQLITE_OK; @@ -74463,7 +74492,7 @@ static int balance_nonroot( iOvflSpace += sz; assert( sz<=pBt->maxLocal+23 ); assert( iOvflSpace <= (int)pBt->pageSize ); - for(k=0; b.ixNx[k]<=i && ALWAYS(kpKeyInfo==0 ); - if( pCur->eState==CURSOR_FAULT ){ - assert( pCur->skipNext!=SQLITE_OK ); - return pCur->skipNext; - } - - assert( cursorOwnsBtShared(pCur) ); - assert( (pCur->curFlags & BTCF_WriteFlag)!=0 - && pBt->inTransaction==TRANS_WRITE - && (pBt->btsFlags & BTS_READ_ONLY)==0 ); - assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); - - /* Assert that the caller has been consistent. If this cursor was opened - ** expecting an index b-tree, then the caller should be inserting blob - ** keys with no associated data. If the cursor was opened expecting an - ** intkey table, the caller should be inserting integer keys with a - ** blob of associated data. */ - assert( (flags & BTREE_PREFORMAT) || (pX->pKey==0)==(pCur->pKeyInfo==0) ); - /* Save the positions of any other cursors open on this table. ** ** In some cases, the call to btreeMoveto() below is a no-op. For @@ -75021,6 +75032,24 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( } } + if( pCur->eState>=CURSOR_REQUIRESEEK ){ + rc = moveToRoot(pCur); + if( rc && rc!=SQLITE_EMPTY ) return rc; + } + + assert( cursorOwnsBtShared(pCur) ); + assert( (pCur->curFlags & BTCF_WriteFlag)!=0 + && pBt->inTransaction==TRANS_WRITE + && (pBt->btsFlags & BTS_READ_ONLY)==0 ); + assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); + + /* Assert that the caller has been consistent. If this cursor was opened + ** expecting an index b-tree, then the caller should be inserting blob + ** keys with no associated data. If the cursor was opened expecting an + ** intkey table, the caller should be inserting integer keys with a + ** blob of associated data. */ + assert( (flags & BTREE_PREFORMAT) || (pX->pKey==0)==(pCur->pKeyInfo==0) ); + if( pCur->pKeyInfo==0 ){ assert( pX->pKey==0 ); /* If this is an insert into a table b-tree, invalidate any incrblob @@ -75109,14 +75138,13 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( } } assert( pCur->eState==CURSOR_VALID - || (pCur->eState==CURSOR_INVALID && loc) - || CORRUPT_DB ); + || (pCur->eState==CURSOR_INVALID && loc) ); pPage = pCur->pPage; assert( pPage->intKey || pX->nKey>=0 || (flags & BTREE_PREFORMAT) ); assert( pPage->leaf || !pPage->intKey ); if( pPage->nFree<0 ){ - if( NEVER(pCur->eState>CURSOR_INVALID) ){ + if( pCur->eState>CURSOR_INVALID ){ rc = SQLITE_CORRUPT_BKPT; }else{ rc = btreeComputeFreeSpace(pPage); @@ -75397,12 +75425,16 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); assert( !hasReadConflicts(p, pCur->pgnoRoot) ); assert( (flags & ~(BTREE_SAVEPOSITION | BTREE_AUXDELETE))==0 ); - if( pCur->eState==CURSOR_REQUIRESEEK ){ - rc = btreeRestoreCursorPosition(pCur); - assert( rc!=SQLITE_OK || CORRUPT_DB || pCur->eState==CURSOR_VALID ); - if( rc || pCur->eState!=CURSOR_VALID ) return rc; + if( pCur->eState!=CURSOR_VALID ){ + if( pCur->eState>=CURSOR_REQUIRESEEK ){ + rc = btreeRestoreCursorPosition(pCur); + assert( rc!=SQLITE_OK || CORRUPT_DB || pCur->eState==CURSOR_VALID ); + if( rc || pCur->eState!=CURSOR_VALID ) return rc; + }else{ + return SQLITE_CORRUPT_BKPT; + } } - assert( CORRUPT_DB || pCur->eState==CURSOR_VALID ); + assert( pCur->eState==CURSOR_VALID ); iCellDepth = pCur->iPage; iCellIdx = pCur->ix; @@ -78025,7 +78057,11 @@ SQLITE_PRIVATE int sqlite3VdbeChangeEncoding(Mem *pMem, int desiredEnc){ assert( !sqlite3VdbeMemIsRowSet(pMem) ); assert( desiredEnc==SQLITE_UTF8 || desiredEnc==SQLITE_UTF16LE || desiredEnc==SQLITE_UTF16BE ); - if( !(pMem->flags&MEM_Str) || pMem->enc==desiredEnc ){ + if( !(pMem->flags&MEM_Str) ){ + pMem->enc = desiredEnc; + return SQLITE_OK; + } + if( pMem->enc==desiredEnc ){ return SQLITE_OK; } assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); @@ -78737,6 +78773,7 @@ SQLITE_PRIVATE void sqlite3VdbeMemSetPointer( void (*xDestructor)(void*) ){ assert( pMem->flags==MEM_Null ); + vdbeMemClear(pMem); pMem->u.zPType = zPType ? zPType : ""; pMem->z = pPtr; pMem->flags = MEM_Null|MEM_Dyn|MEM_Subtype|MEM_Term; @@ -88351,6 +88388,8 @@ case OP_Gosub: { /* jump */ /* Most jump operations do a goto to this spot in order to update ** the pOp pointer. */ jump_to_p2: + assert( pOp->p2>0 ); /* There are never any jumps to instruction 0 */ + assert( pOp->p2nOp ); /* Jumps must be in range */ pOp = &aOp[pOp->p2 - 1]; break; } @@ -89991,10 +90030,18 @@ case OP_Offset: { /* out3 */ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; pOut = &p->aMem[pOp->p3]; - if( NEVER(pC==0) || pC->eCurType!=CURTYPE_BTREE ){ + if( pC==0 || pC->eCurType!=CURTYPE_BTREE ){ sqlite3VdbeMemSetNull(pOut); }else{ - sqlite3VdbeMemSetInt64(pOut, sqlite3BtreeOffset(pC->uc.pCursor)); + if( pC->deferredMoveto ){ + rc = sqlite3VdbeFinishMoveto(pC); + if( rc ) goto abort_due_to_error; + } + if( sqlite3BtreeEof(pC->uc.pCursor) ){ + sqlite3VdbeMemSetNull(pOut); + }else{ + sqlite3VdbeMemSetInt64(pOut, sqlite3BtreeOffset(pC->uc.pCursor)); + } } break; } @@ -93073,6 +93120,10 @@ case OP_Rowid: { /* out2 */ ** Move the cursor P1 to a null row. Any OP_Column operations ** that occur while the cursor is on the null row will always ** write a NULL. +** +** Or, if P1 is a Pseudo-Cursor (a cursor opened using OP_OpenPseudo) +** just reset the cache for that cursor. This causes the row of +** content held by the pseudo-cursor to be reparsed. */ case OP_NullRow: { VdbeCursor *pC; @@ -104719,6 +104770,38 @@ SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr *p, int iCur){ return exprIsConst(p, 3, iCur); } +/* +** Check pExpr to see if it is an invariant constraint on data source pSrc. +** This is an optimization. False negatives will perhaps cause slower +** queries, but false positives will yield incorrect answers. So when in +** double, return 0. +** +** To be an invariant constraint, the following must be true: +** +** (1) pExpr cannot refer to any table other than pSrc->iCursor. +** +** (2) pExpr cannot use subqueries or non-deterministic functions. +** +** (*) ** Not applicable to this branch ** +** +** (4) If pSrc is the right operand of a LEFT JOIN, then... +** (4a) pExpr must come from an ON clause.. +** (4b) and specifically the ON clause associated with the LEFT JOIN. +** +** (5) If pSrc is not the right operand of a LEFT JOIN or the left +** operand of a RIGHT JOIN, then pExpr must be from the WHERE +** clause, not an ON clause. +*/ +SQLITE_PRIVATE int sqlite3ExprIsTableConstraint(Expr *pExpr, const SrcItem *pSrc){ + if( pSrc->fg.jointype & JT_LEFT ){ + if( !ExprHasProperty(pExpr, EP_FromJoin) ) return 0; /* rule (4a) */ + if( pExpr->w.iRightJoinTable!=pSrc->iCursor ) return 0; /* rule (4b) */ + }else{ + if( ExprHasProperty(pExpr, EP_FromJoin) ) return 0; /* rule (5) */ + } + return sqlite3ExprIsTableConstant(pExpr, pSrc->iCursor); /* rules (1), (2) */ +} + /* ** sqlite3WalkExpr() callback used by sqlite3ExprIsConstantOrGroupBy(). @@ -109919,19 +110002,21 @@ static int renameParseSql( ){ int rc; - db->init.iDb = bTemp ? 1 : sqlite3FindDbName(db, zDb); - - /* Parse the SQL statement passed as the first argument. If no error - ** occurs and the parse does not result in a new table, index or - ** trigger object, the database must be corrupt. */ sqlite3ParseObjectInit(p, db); + if( zSql==0 ){ + return SQLITE_NOMEM; + } + if( sqlite3StrNICmp(zSql,"CREATE ",7)!=0 ){ + return SQLITE_CORRUPT_BKPT; + } + db->init.iDb = bTemp ? 1 : sqlite3FindDbName(db, zDb); p->eParseMode = PARSE_MODE_RENAME; p->db = db; p->nQueryLoop = 1; - rc = zSql ? sqlite3RunParser(p, zSql) : SQLITE_NOMEM; + rc = sqlite3RunParser(p, zSql); if( db->mallocFailed ) rc = SQLITE_NOMEM; if( rc==SQLITE_OK - && p->pNewTable==0 && p->pNewIndex==0 && p->pNewTrigger==0 + && NEVER(p->pNewTable==0 && p->pNewIndex==0 && p->pNewTrigger==0) ){ rc = SQLITE_CORRUPT_BKPT; } @@ -116680,6 +116765,11 @@ SQLITE_PRIVATE void sqlite3EndTable( int addrInsLoop; /* Top of the loop for inserting rows */ Table *pSelTab; /* A table that describes the SELECT results */ + if( IN_SPECIAL_PARSE ){ + pParse->rc = SQLITE_ERROR; + pParse->nErr++; + return; + } regYield = ++pParse->nMem; regRec = ++pParse->nMem; regRowid = ++pParse->nMem; @@ -123259,8 +123349,8 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ INLINE_FUNC(likelihood, 2, INLINEFUNC_unlikely, SQLITE_FUNC_UNLIKELY), INLINE_FUNC(likely, 1, INLINEFUNC_unlikely, SQLITE_FUNC_UNLIKELY), #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC - FUNCTION2(sqlite_offset, 1, 0, 0, noopFunc, SQLITE_FUNC_OFFSET| - SQLITE_FUNC_TYPEOF), + {1, SQLITE_FUNC_BUILTIN|SQLITE_UTF8|SQLITE_FUNC_OFFSET|SQLITE_FUNC_TYPEOF, + 0, 0, noopFunc, 0, 0, 0, "sqlite_offset", {0} }, #endif FUNCTION(ltrim, 1, 1, 0, trimFunc ), FUNCTION(ltrim, 2, 1, 0, trimFunc ), @@ -125064,7 +125154,7 @@ SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ } for(i=j=0; inCol; i++){ - assert( pTab->aCol[i].affinity!=0 ); + assert( pTab->aCol[i].affinity!=0 || sqlite3VdbeParser(v)->nErr>0 ); if( (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 ){ zColAff[j++] = pTab->aCol[i].affinity; } @@ -125678,7 +125768,11 @@ SQLITE_PRIVATE void sqlite3Insert( ** ** This is the 2nd template. */ - if( pColumn==0 && xferOptimization(pParse, pTab, pSelect, onError, iDb) ){ + if( pColumn==0 + && pSelect!=0 + && pTrigger==0 + && xferOptimization(pParse, pTab, pSelect, onError, iDb) + ){ assert( !pTrigger ); assert( pList==0 ); goto insert_end; @@ -127649,18 +127743,13 @@ static int xferOptimization( int destHasUniqueIdx = 0; /* True if pDest has a UNIQUE index */ int regData, regRowid; /* Registers holding data and rowid */ - if( pSelect==0 ){ - return 0; /* Must be of the form INSERT INTO ... SELECT ... */ - } + assert( pSelect!=0 ); if( pParse->pWith || pSelect->pWith ){ /* Do not attempt to process this query if there are an WITH clauses ** attached to it. Proceeding may generate a false "no such table: xxx" ** error if pSelect reads from a CTE named "xxx". */ return 0; } - if( sqlite3TriggerList(pParse, pDest) ){ - return 0; /* tab1 must not have triggers */ - } #ifndef SQLITE_OMIT_VIRTUALTABLE if( IsVirtual(pDest) ){ return 0; /* tab1 must not be a virtual table */ @@ -133506,7 +133595,7 @@ SQLITE_PRIVATE int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg, u32 mFl sqlite3ResetAllSchemasOfConnection(db); pDb = &db->aDb[iDb]; }else - if( rc==SQLITE_OK || (db->flags&SQLITE_NoSchemaError)){ + if( rc==SQLITE_OK || ((db->flags&SQLITE_NoSchemaError) && rc!=SQLITE_NOMEM)){ /* Hack: If the SQLITE_NoSchemaError flag is set, then consider ** the schema loaded, even if errors (other than OOM) occurred. In ** this situation the current sqlite3_prepare() operation will fail, @@ -133780,6 +133869,14 @@ SQLITE_PRIVATE void sqlite3ParseObjectInit(Parse *pParse, sqlite3 *db){ if( db->mallocFailed ) sqlite3ErrorMsg(pParse, "out of memory"); } +/* +** Maximum number of times that we will try again to prepare a statement +** that returns SQLITE_ERROR_RETRY. +*/ +#ifndef SQLITE_MAX_PREPARE_RETRY +# define SQLITE_MAX_PREPARE_RETRY 25 +#endif + /* ** Compile the UTF-8 encoded SQL statement zSql into a statement handle. */ @@ -133954,7 +134051,7 @@ static int sqlite3LockAndPrepare( rc = sqlite3Prepare(db, zSql, nBytes, prepFlags, pOld, ppStmt, pzTail); assert( rc==SQLITE_OK || *ppStmt==0 ); if( rc==SQLITE_OK || db->mallocFailed ) break; - }while( rc==SQLITE_ERROR_RETRY + }while( (rc==SQLITE_ERROR_RETRY && (cnt++)op==TK_AND ){ - nChng += pushDownWhereTerms(pParse, pSubq, pWhere->pRight, - iCursor, isLeftJoin); + nChng += pushDownWhereTerms(pParse, pSubq, pWhere->pRight, pSrc); pWhere = pWhere->pLeft; } + +#if 0 /* Legacy code. Checks now done by sqlite3ExprIsTableConstraint() */ if( isLeftJoin && (ExprHasProperty(pWhere,EP_FromJoin)==0 || pWhere->w.iRightJoinTable!=iCursor) @@ -139047,7 +139144,9 @@ static int pushDownWhereTerms( ){ return 0; /* restriction (5) */ } - if( sqlite3ExprIsTableConstant(pWhere, iCursor) ){ +#endif + + if( sqlite3ExprIsTableConstraint(pWhere, pSrc) ){ nChng++; pSubq->selFlags |= SF_PushDown; while( pSubq ){ @@ -139055,8 +139154,8 @@ static int pushDownWhereTerms( pNew = sqlite3ExprDup(pParse->db, pWhere, 0); unsetJoinExpr(pNew, -1); x.pParse = pParse; - x.iTable = iCursor; - x.iNewTable = iCursor; + x.iTable = pSrc->iCursor; + x.iNewTable = pSrc->iCursor; x.isLeftJoin = 0; x.pEList = pSubq->pEList; pNew = substExpr(&x, pNew); @@ -140838,8 +140937,7 @@ SQLITE_PRIVATE int sqlite3Select( if( OptimizationEnabled(db, SQLITE_PushDown) && (pItem->fg.isCte==0 || (pItem->u2.pCteUse->eM10d!=M10d_Yes && pItem->u2.pCteUse->nUse<2)) - && pushDownWhereTerms(pParse, pSub, p->pWhere, pItem->iCursor, - (pItem->fg.jointype & JT_OUTER)!=0) + && pushDownWhereTerms(pParse, pSub, p->pWhere, pItem) ){ #if SELECTTRACE_ENABLED if( sqlite3SelectTrace & 0x100 ){ @@ -142373,6 +142471,7 @@ static TriggerStep *triggerStepAllocate( sqlite3 *db = pParse->db; TriggerStep *pTriggerStep; + if( pParse->nErr ) return 0; pTriggerStep = sqlite3DbMallocZero(db, sizeof(TriggerStep) + pName->n + 1); if( pTriggerStep ){ char *z = (char*)&pTriggerStep[1]; @@ -146251,6 +146350,7 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ sqlite3ParseObjectInit(&sParse, db); sParse.eParseMode = PARSE_MODE_DECLARE_VTAB; + sParse.disableTriggers = 1; /* We should never be able to reach this point while loading the ** schema. Nevertheless, defend against that (turn off db->init.busy) ** in case a bug arises. */ @@ -148186,6 +148286,7 @@ static int codeAllEqualityTerms( VdbeCoverageIf(v, bRev!=0); VdbeComment((v, "begin skip-scan on %s", pIdx->zName)); j = sqlite3VdbeAddOp0(v, OP_Goto); + assert( pLevel->addrSkip==0 ); pLevel->addrSkip = sqlite3VdbeAddOp4Int(v, (bRev?OP_SeekLT:OP_SeekGT), iIdxCur, 0, regBase, nSkip); VdbeCoverageIf(v, bRev==0); @@ -148628,6 +148729,7 @@ static void preserveExpr(IdxExprTrans *pTrans, Expr *pExpr){ static int whereIndexExprTransNode(Walker *p, Expr *pExpr){ IdxExprTrans *pX = p->u.pIdxTrans; if( sqlite3ExprCompare(0, pExpr, pX->pIdxExpr, pX->iTabCur)==0 ){ + pExpr = sqlite3ExprSkipCollate(pExpr); preserveExpr(pX, pExpr); pExpr->affExpr = sqlite3ExprAffinity(pExpr); pExpr->op = TK_COLUMN; @@ -148783,9 +148885,12 @@ static SQLITE_NOINLINE void filterPullDown( WhereLevel *pLevel = &pWInfo->a[iLevel]; WhereLoop *pLoop = pLevel->pWLoop; if( pLevel->regFilter==0 ) continue; + if( pLevel->pWLoop->nSkip ) continue; /* ,--- Because sqlite3ConstructBloomFilter() has will not have set ** vvvvv--' pLevel->regFilter if this were true. */ if( NEVER(pLoop->prereq & notReady) ) continue; + assert( pLevel->addrBrk==0 ); + pLevel->addrBrk = addrNxt; if( pLoop->wsFlags & WHERE_IPK ){ WhereTerm *pTerm = pLoop->aLTerm[0]; int regRowid; @@ -148812,6 +148917,7 @@ static SQLITE_NOINLINE void filterPullDown( VdbeCoverage(pParse->pVdbe); } pLevel->regFilter = 0; + pLevel->addrBrk = 0; } } @@ -148917,7 +149023,6 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( int iReg; /* P3 Value for OP_VFilter */ int addrNotFound; int nConstraint = pLoop->nLTerm; - int iIn; /* Counter for IN constraints */ iReg = sqlite3GetTempRange(pParse, nConstraint+2); addrNotFound = pLevel->addrBrk; @@ -148963,50 +149068,54 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( pLevel->op = pWInfo->eOnePass ? OP_Noop : OP_VNext; pLevel->p2 = sqlite3VdbeCurrentAddr(v); assert( (pLoop->wsFlags & WHERE_MULTI_OR)==0 ); - if( pLoop->wsFlags & WHERE_IN_ABLE ){ - iIn = pLevel->u.in.nIn; - }else{ - iIn = 0; - } - for(j=nConstraint-1; j>=0; j--){ - int bIn; /* True to generate byte code to loop over RHS IN values */ + + for(j=0; jaLTerm[j]; + if( j<16 && (pLoop->u.vtab.omitMask>>j)&1 ){ + disableTerm(pLevel, pTerm); + continue; + } if( (pTerm->eOperator & WO_IN)!=0 && (SMASKBIT32(j) & pLoop->u.vtab.mHandleIn)==0 + && !db->mallocFailed ){ - bIn = 1; - }else{ - bIn = 0; - } - if( bIn ) iIn--; - if( j<16 && (pLoop->u.vtab.omitMask>>j)&1 ){ - disableTerm(pLevel, pTerm); - }else if( bIn && sqlite3ExprVectorSize(pTerm->pExpr->pLeft)==1 ){ Expr *pCompare; /* The comparison operator */ Expr *pRight; /* RHS of the comparison */ VdbeOp *pOp; /* Opcode to access the value of the IN constraint */ + int iIn; /* IN loop corresponding to the j-th constraint */ /* Reload the constraint value into reg[iReg+j+2]. The same value ** was loaded into the same register prior to the OP_VFilter, but ** the xFilter implementation might have changed the datatype or - ** encoding of the value in the register, so it *must* be reloaded. */ - assert( pLevel->u.in.aInLoop!=0 || db->mallocFailed ); - if( !db->mallocFailed ){ - assert( iIn>=0 && iInu.in.nIn ); + ** encoding of the value in the register, so it *must* be reloaded. + */ + for(iIn=0; ALWAYS(iInu.in.nIn); iIn++){ pOp = sqlite3VdbeGetOp(v, pLevel->u.in.aInLoop[iIn].addrInTop); - assert( pOp->opcode==OP_Column || pOp->opcode==OP_Rowid ); - assert( pOp->opcode!=OP_Column || pOp->p3==iReg+j+2 ); - assert( pOp->opcode!=OP_Rowid || pOp->p2==iReg+j+2 ); - testcase( pOp->opcode==OP_Rowid ); - sqlite3VdbeAddOp3(v, pOp->opcode, pOp->p1, pOp->p2, pOp->p3); + if( (pOp->opcode==OP_Column && pOp->p3==iReg+j+2) + || (pOp->opcode==OP_Rowid && pOp->p2==iReg+j+2) + ){ + testcase( pOp->opcode==OP_Rowid ); + sqlite3VdbeAddOp3(v, pOp->opcode, pOp->p1, pOp->p2, pOp->p3); + break; + } } /* Generate code that will continue to the next row if - ** the IN constraint is not satisfied */ + ** the IN constraint is not satisfied + */ pCompare = sqlite3PExpr(pParse, TK_EQ, 0, 0); - assert( pCompare!=0 || db->mallocFailed ); - if( pCompare ){ - pCompare->pLeft = pTerm->pExpr->pLeft; + if( !db->mallocFailed ){ + int iFld = pTerm->u.x.iField; + Expr *pLeft = pTerm->pExpr->pLeft; + assert( pLeft!=0 ); + if( iFld>0 ){ + assert( pLeft->op==TK_VECTOR ); + assert( ExprUseXList(pLeft) ); + assert( iFld<=pLeft->x.pList->nExpr ); + pCompare->pLeft = pLeft->x.pList->a[iFld-1].pExpr; + }else{ + pCompare->pLeft = pLeft; + } pCompare->pRight = pRight = sqlite3Expr(db, TK_REGISTER, 0); if( pRight ){ pRight->iTable = iReg+j+2; @@ -149015,11 +149124,11 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ); } pCompare->pLeft = 0; - sqlite3ExprDelete(db, pCompare); } + sqlite3ExprDelete(db, pCompare); } } - assert( iIn==0 || db->mallocFailed ); + /* These registers need to be preserved in case there is an IN operator ** loop. So we could deallocate the registers here (and potentially ** reuse them later) if (pLoop->wsFlags & WHERE_IN_ABLE)==0. But it seems @@ -149729,6 +149838,14 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** the initialization of the right-hand operand of the vector comparison ** might not occur, or might occur only in an OR branch that is not ** taken. dbsqlfuzz 80a9fade844b4fb43564efc972bcb2c68270f5d1. + ** + ** 2022-03-03: Do not push down expressions that involve subqueries. + ** The subquery might get coded as a subroutine. Any table-references + ** in the subquery might be resolved to index-references for the index on + ** the OR branch in which the subroutine is coded. But if the subroutine + ** is invoked from a different OR branch that uses a different index, such + ** index-references will not work. tag-20220303a + ** https://sqlite.org/forum/forumpost/36937b197273d403 */ if( pWC->nTerm>1 ){ int iTerm; @@ -149742,7 +149859,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( continue; } if( (pWC->a[iTerm].eOperator & WO_ALL)==0 ) continue; - testcase( pWC->a[iTerm].wtFlags & TERM_ORINFO ); + if( ExprHasProperty(pExpr, EP_Subquery) ) continue; /* tag-20220303a */ pExpr = sqlite3ExprDup(db, pExpr, 0); pAndExpr = sqlite3ExprAnd(pParse, pAndExpr, pExpr); } @@ -152749,8 +152866,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex( ** WHERE clause (or the ON clause of a LEFT join) that constrain which ** rows of the target table (pSrc) that can be used. */ if( (pTerm->wtFlags & TERM_VIRTUAL)==0 - && ((pSrc->fg.jointype&JT_LEFT)==0 || ExprHasProperty(pExpr,EP_FromJoin)) - && sqlite3ExprIsTableConstant(pExpr, pSrc->iCursor) + && sqlite3ExprIsTableConstraint(pExpr, pSrc) ){ pPartial = sqlite3ExprAnd(pParse, pPartial, sqlite3ExprDup(pParse->db, pExpr, 0)); @@ -152989,7 +153105,7 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( for(pTerm=pWInfo->sWC.a; pTermpExpr; if( (pTerm->wtFlags & TERM_VIRTUAL)==0 - && sqlite3ExprIsTableConstant(pExpr, iCur) + && sqlite3ExprIsTableConstraint(pExpr, pItem) ){ sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL); } @@ -153019,7 +153135,10 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( pLoop->wsFlags &= ~WHERE_BLOOMFILTER; if( OptimizationDisabled(pParse->db, SQLITE_BloomPulldown) ) break; while( ++iLevel < pWInfo->nLevel ){ + const SrcItem *pTabItem; pLevel = &pWInfo->a[iLevel]; + pTabItem = &pWInfo->pTabList->a[pLevel->iFrom]; + if( pTabItem->fg.jointype & JT_LEFT ) continue; pLoop = pLevel->pWLoop; if( NEVER(pLoop==0) ) continue; if( pLoop->prereq & notReady ) continue; @@ -154532,8 +154651,17 @@ static void whereLoopOutputAdjust( /* If there are extra terms in the WHERE clause not used by an index ** that depend only on the table being scanned, and that will tend to ** cause many rows to be omitted, then mark that table as - ** "self-culling". */ - pLoop->wsFlags |= WHERE_SELFCULL; + ** "self-culling". + ** + ** 2022-03-24: Self-culling only applies if either the extra terms + ** are straight comparison operators that are non-true with NULL + ** operand, or if the loop is not a LEFT JOIN. + */ + if( (pTerm->eOperator & 0x3f)!=0 + || (pWC->pWInfo->pTabList->a[pLoop->iTab].fg.jointype & JT_LEFT)==0 + ){ + pLoop->wsFlags |= WHERE_SELFCULL; + } } if( pTerm->truthProb<=0 ){ /* If a truth probability is specified using the likelihood() hints, @@ -155702,7 +155830,6 @@ SQLITE_API int sqlite3_vtab_rhs_value( return rc; } - /* ** Return true if ORDER BY clause may be handled as DISTINCT. */ @@ -155714,6 +155841,22 @@ SQLITE_API int sqlite3_vtab_distinct(sqlite3_index_info *pIdxInfo){ return pHidden->eDistinct; } +#if (defined(SQLITE_ENABLE_DBPAGE_VTAB) || defined(SQLITE_TEST)) \ + && !defined(SQLITE_OMIT_VIRTUALTABLE) +/* +** Cause the prepared statement that is associated with a call to +** xBestIndex to open write transactions on all attached schemas. +** This is used by the (built-in) sqlite_dbpage virtual table. +*/ +SQLITE_PRIVATE void sqlite3VtabWriteAll(sqlite3_index_info *pIdxInfo){ + HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1]; + Parse *pParse = pHidden->pParse; + int nDb = pParse->db->nDb; + int i; + for(i=0; ipNew->iTab. That table is guaranteed to be a virtual table. @@ -157817,6 +157960,26 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( } #endif +#ifdef SQLITE_DEBUG +/* +** Return true if cursor iCur is opened by instruction k of the +** bytecode. Used inside of assert() only. +*/ +static int cursorIsOpen(Vdbe *v, int iCur, int k){ + while( k>=0 ){ + VdbeOp *pOp = sqlite3VdbeGetOp(v,k--); + if( pOp->p1!=iCur ) continue; + if( pOp->opcode==OP_Close ) return 0; + if( pOp->opcode==OP_OpenRead ) return 1; + if( pOp->opcode==OP_OpenWrite ) return 1; + if( pOp->opcode==OP_OpenDup ) return 1; + if( pOp->opcode==OP_OpenAutoindex ) return 1; + if( pOp->opcode==OP_OpenEphemeral ) return 1; + } + return 0; +} +#endif /* SQLITE_DEBUG */ + /* ** Generate the end of the WHERE loop. See comments on ** sqlite3WhereBegin() for additional information. @@ -158069,6 +158232,11 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ ){ int x = pOp->p2; assert( pIdx->pTable==pTab ); +#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC + if( pOp->opcode==OP_Offset ){ + /* Do not need to translate the column number */ + }else +#endif if( !HasRowid(pTab) ){ Index *pPk = sqlite3PrimaryKeyIndex(pTab); x = pPk->aiColumn[x]; @@ -158082,9 +158250,22 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ pOp->p2 = x; pOp->p1 = pLevel->iIdxCur; OpcodeRewriteTrace(db, k, pOp); + }else{ + /* Unable to translate the table reference into an index + ** reference. Verify that this is harmless - that the + ** table being referenced really is open. + */ +#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC + assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 + || cursorIsOpen(v,pOp->p1,k) + || pOp->opcode==OP_Offset + ); +#else + assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 + || cursorIsOpen(v,pOp->p1,k) + ); +#endif } - assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 || x>=0 - || pWInfo->eOnePass ); }else if( pOp->opcode==OP_Rowid ){ pOp->p1 = pLevel->iIdxCur; pOp->opcode = OP_IdxRowid; @@ -159072,7 +159253,11 @@ static int disallowAggregatesInOrderByCb(Walker *pWalker, Expr *pExpr){ */ SQLITE_PRIVATE int sqlite3WindowRewrite(Parse *pParse, Select *p){ int rc = SQLITE_OK; - if( p->pWin && p->pPrior==0 && ALWAYS((p->selFlags & SF_WinRewrite)==0) ){ + if( p->pWin + && p->pPrior==0 + && ALWAYS((p->selFlags & SF_WinRewrite)==0) + && ALWAYS(!IN_RENAME_OBJECT) + ){ Vdbe *v = sqlite3GetVdbe(pParse); sqlite3 *db = pParse->db; Select *pSub = 0; /* The subquery */ @@ -159147,6 +159332,7 @@ SQLITE_PRIVATE int sqlite3WindowRewrite(Parse *pParse, Select *p){ for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ ExprList *pArgs; assert( ExprUseXList(pWin->pOwner) ); + assert( pWin->pFunc!=0 ); pArgs = pWin->pOwner->x.pList; if( pWin->pFunc->funcFlags & SQLITE_FUNC_SUBTYPE ){ selectWindowRewriteEList(pParse, pMWin, pSrc, pArgs, pTab, &pSublist); @@ -159839,7 +160025,7 @@ static void windowAggStep( for(iEnd=sqlite3VdbeCurrentAddr(v); iOpopcode==OP_Column && pOp->p1==pWin->iEphCsr ){ + if( pOp->opcode==OP_Column && pOp->p1==pMWin->iEphCsr ){ pOp->p1 = csr; } } @@ -194157,14 +194343,15 @@ static JsonNode *jsonLookupStep( *pzErr = zPath; return 0; } + testcase( nKey==0 ); }else{ zKey = zPath; for(i=0; zPath[i] && zPath[i]!='.' && zPath[i]!='['; i++){} nKey = i; - } - if( nKey==0 ){ - *pzErr = zPath; - return 0; + if( nKey==0 ){ + *pzErr = zPath; + return 0; + } } j = 1; for(;;){ @@ -195312,6 +195499,33 @@ static int jsonEachNext(sqlite3_vtab_cursor *cur){ return SQLITE_OK; } +/* Append an object label to the JSON Path being constructed +** in pStr. +*/ +static void jsonAppendObjectPathElement( + JsonString *pStr, + JsonNode *pNode +){ + int jj, nn; + const char *z; + assert( pNode->eType==JSON_STRING ); + assert( pNode->jnFlags & JNODE_LABEL ); + assert( pNode->eU==1 ); + z = pNode->u.zJContent; + nn = pNode->n; + assert( nn>=2 ); + assert( z[0]=='"' ); + assert( z[nn-1]=='"' ); + if( nn>2 && sqlite3Isalpha(z[1]) ){ + for(jj=2; jjeType==JSON_OBJECT ); if( (pNode->jnFlags & JNODE_LABEL)==0 ) pNode--; - assert( pNode->eType==JSON_STRING ); - assert( pNode->jnFlags & JNODE_LABEL ); - assert( pNode->eU==1 ); - jsonPrintf(pNode->n+1, pStr, ".%.*s", pNode->n-2, pNode->u.zJContent+1); + jsonAppendObjectPathElement(pStr, pNode); } } @@ -195410,8 +195621,7 @@ static int jsonEachColumn( if( p->eType==JSON_ARRAY ){ jsonPrintf(30, &x, "[%d]", p->iRowid); }else if( p->eType==JSON_OBJECT ){ - assert( pThis->eU==1 ); - jsonPrintf(pThis->n, &x, ".%.*s", pThis->n-2, pThis->u.zJContent+1); + jsonAppendObjectPathElement(&x, pThis); } } jsonResult(&x); @@ -209958,6 +210168,7 @@ static int dbpageBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ ){ pIdxInfo->orderByConsumed = 1; } + sqlite3VtabWriteAll(pIdxInfo); return SQLITE_OK; } @@ -232312,7 +232523,7 @@ static int fts5SorterNext(Fts5Cursor *pCsr){ rc = sqlite3_step(pSorter->pStmt); if( rc==SQLITE_DONE ){ rc = SQLITE_OK; - CsrFlagSet(pCsr, FTS5CSR_EOF); + CsrFlagSet(pCsr, FTS5CSR_EOF|FTS5CSR_REQUIRE_CONTENT); }else if( rc==SQLITE_ROW ){ const u8 *a; const u8 *aBlob; @@ -234301,7 +234512,7 @@ static void fts5SourceIdFunc( ){ assert( nArg==0 ); UNUSED_PARAM2(nArg, apUnused); - sqlite3_result_text(pCtx, "fts5: 2022-02-22 18:58:40 40fa792d359f84c3b9e9d6623743e1a59826274e221df1bde8f47086968a1bab", -1, SQLITE_TRANSIENT); + sqlite3_result_text(pCtx, "fts5: 2022-05-06 15:25:27 78d9c993d404cdfaa7fdd2973fa1052e3da9f66215cff9c5540ebe55c407d9fe", -1, SQLITE_TRANSIENT); } /* diff --git a/sqlite3-binding.h b/sqlite3-binding.h index 20bc9780..1c267a40 100644 --- a/sqlite3-binding.h +++ b/sqlite3-binding.h @@ -147,9 +147,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.38.0" -#define SQLITE_VERSION_NUMBER 3038000 -#define SQLITE_SOURCE_ID "2022-02-22 18:58:40 40fa792d359f84c3b9e9d6623743e1a59826274e221df1bde8f47086968a1bab" +#define SQLITE_VERSION "3.38.5" +#define SQLITE_VERSION_NUMBER 3038005 +#define SQLITE_SOURCE_ID "2022-05-06 15:25:27 78d9c993d404cdfaa7fdd2973fa1052e3da9f66215cff9c5540ebe55c407d9fe" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -4980,6 +4980,10 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** even empty strings, are always zero-terminated. ^The return ** value from sqlite3_column_blob() for a zero-length BLOB is a NULL pointer. ** +** ^Strings returned by sqlite3_column_text16() always have the endianness +** which is native to the platform, regardless of the text encoding set +** for the database. +** ** Warning: ^The object returned by [sqlite3_column_value()] is an ** [unprotected sqlite3_value] object. In a multithreaded environment, ** an unprotected sqlite3_value object may only be used safely with @@ -4993,7 +4997,7 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** [application-defined SQL functions] or [virtual tables], not within ** top-level application code. ** -** The these routines may attempt to convert the datatype of the result. +** These routines may attempt to convert the datatype of the result. ** ^For example, if the internal representation is FLOAT and a text result ** is requested, [sqlite3_snprintf()] is used internally to perform the ** conversion automatically. ^(The following table details the conversions @@ -5018,7 +5022,7 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** TEXT BLOB No change ** BLOB INTEGER [CAST] to INTEGER ** BLOB FLOAT [CAST] to REAL -** BLOB TEXT Add a zero terminator if needed +** BLOB TEXT [CAST] to TEXT, ensure zero terminator ** ** )^ ** @@ -9768,7 +9772,7 @@ SQLITE_API int sqlite3_vtab_in_next(sqlite3_value *pVal, sqlite3_value **ppOut); ** ^When xBestIndex returns, the sqlite3_value object returned by ** sqlite3_vtab_rhs_value() is automatically deallocated. ** -** The "_rhs_" in the name of this routine is an appreviation for +** The "_rhs_" in the name of this routine is an abbreviation for ** "Right-Hand Side". */ SQLITE_API int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value **ppVal); diff --git a/sqlite3_opt_json1.go b/sqlite3_opt_json1.go deleted file mode 100644 index 7cfce763..00000000 --- a/sqlite3_opt_json1.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (C) 2019 Yasuhiro Matsumoto . -// -// Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file. - -// +build sqlite_json sqlite_json1 json1 - -package sqlite3 - -/* -#cgo CFLAGS: -DSQLITE_ENABLE_JSON1 -*/ -import "C" diff --git a/sqlite3_opt_preupdate_hook.go b/sqlite3_opt_preupdate_hook.go index b2e18bbc..b43e4821 100644 --- a/sqlite3_opt_preupdate_hook.go +++ b/sqlite3_opt_preupdate_hook.go @@ -33,7 +33,7 @@ import ( // The callback is passed a SQLitePreUpdateData struct with the data for // the update, as well as methods for fetching copies of impacted data. // -// If there is an existing update hook for this connection, it will be +// If there is an existing preupdate hook for this connection, it will be // removed. If callback is nil the existing hook (if any) will be removed // without creating a new one. func (c *SQLiteConn) RegisterPreUpdateHook(callback func(SQLitePreUpdateData)) { diff --git a/sqlite3_opt_preupdate_omit.go b/sqlite3_opt_preupdate_omit.go index 8df453de..c510a15b 100644 --- a/sqlite3_opt_preupdate_omit.go +++ b/sqlite3_opt_preupdate_omit.go @@ -13,7 +13,7 @@ package sqlite3 // The callback is passed a SQLitePreUpdateData struct with the data for // the update, as well as methods for fetching copies of impacted data. // -// If there is an existing update hook for this connection, it will be +// If there is an existing preupdate hook for this connection, it will be // removed. If callback is nil the existing hook (if any) will be removed // without creating a new one. func (c *SQLiteConn) RegisterPreUpdateHook(callback func(SQLitePreUpdateData)) { From 0613db8db806ad537e7711ad65a5bb7c020a2c2f Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Wed, 26 Oct 2022 20:08:51 -0400 Subject: [PATCH 17/39] Update CircleCI image --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 443c312d..a949e7e1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ jobs: test: docker: # specify the version - - image: circleci/golang:1.14 + - image: cimg/go:1.18.0 # Specify service dependencies here if necessary # CircleCI maintains a library of pre-built images @@ -23,6 +23,7 @@ jobs: - checkout # specify any bash command here prefixed with `run: ` + - run: go version - run: go get -v -t -d ./... - run: go vet - run: go test -v ./... From 7d4a77883557e056249586bfc943754a4a42eeef Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Wed, 26 Oct 2022 20:10:02 -0400 Subject: [PATCH 18/39] Remove obsolete directive --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a949e7e1..cd7b0757 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,7 +18,6 @@ jobs: #### expecting it in the form of #### /go/src/github.com/circleci/go-tool #### /go/src/bitbucket.org/circleci/go-tool - working_directory: /go/src/github.com/rqlite/go-sqlite3 steps: - checkout From 44238f2e2320a99fe7b0aa6259b0f8799f9503f2 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Fri, 18 Nov 2022 09:28:58 -0500 Subject: [PATCH 19/39] Fix go vet issue 'possible misuse of reflect.SliceHeader' --- sqlite3_opt_serialize.go | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/sqlite3_opt_serialize.go b/sqlite3_opt_serialize.go index 2560c43a..845bde82 100644 --- a/sqlite3_opt_serialize.go +++ b/sqlite3_opt_serialize.go @@ -42,11 +42,9 @@ func (c *SQLiteConn) Serialize(schema string) ([]byte, error) { return nil, fmt.Errorf("serialized database is too large (%d bytes)", sz) } - cBuf := *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{ - Data: uintptr(unsafe.Pointer(ptr)), - Len: int(sz), - Cap: int(sz), - })) + cBuf := make([]byte, int(sz)) + sh := (*reflect.SliceHeader)(unsafe.Pointer(&cBuf)) + sh.Data = uintptr(unsafe.Pointer(ptr)) res := make([]byte, int(sz)) copy(res, cBuf) @@ -65,12 +63,11 @@ func (c *SQLiteConn) Deserialize(b []byte, schema string) error { zSchema = C.CString(schema) defer C.free(unsafe.Pointer(zSchema)) + tmpBuf := (*C.uchar)(C.sqlite3_malloc64(C.sqlite3_uint64(len(b)))) - cBuf := *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{ - Data: uintptr(unsafe.Pointer(tmpBuf)), - Len: len(b), - Cap: len(b), - })) + cBuf := make([]byte, len(b)) + sh := (*reflect.SliceHeader)(unsafe.Pointer(&cBuf)) + sh.Data = uintptr(unsafe.Pointer(tmpBuf)) copy(cBuf, b) rc := C.sqlite3_deserialize(c.db, zSchema, tmpBuf, C.sqlite3_int64(len(b)), From 4dde487be162d653b748f14b608abdd4ac834cbf Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Mon, 5 Dec 2022 21:10:03 -0500 Subject: [PATCH 20/39] Fix in-memory database locking https://sqlite.org/src/info/d18cce37b5b73bb2 --- sqlite3-binding.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/sqlite3-binding.c b/sqlite3-binding.c index dd05f575..5c589890 100644 --- a/sqlite3-binding.c +++ b/sqlite3-binding.c @@ -50141,15 +50141,28 @@ static int memdbFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ ** Lock an memdb-file. */ static int memdbLock(sqlite3_file *pFile, int eLock){ - MemFile *pThis = (MemFile*)pFile; + MemFile *pThis = (MemFile*)pFile; MemStore *p = pThis->pStore; int rc = SQLITE_OK; if( eLock==pThis->eLock ) return SQLITE_OK; memdbEnter(p); if( eLock>SQLITE_LOCK_SHARED ){ + assert( pThis->eLock>=SQLITE_LOCK_SHARED ); if( p->mFlags & SQLITE_DESERIALIZE_READONLY ){ rc = SQLITE_READONLY; + }else if( eLock==SQLITE_LOCK_EXCLUSIVE ){ + /* Taking an EXCLUSIVE lock. Fail if we only have SHARED and any + ** other client has any kind of write-lock. Also fail if any other + ** client is holding read-lock. */ + if( pThis->eLock<=SQLITE_LOCK_SHARED && p->nWrLock ){ + rc = SQLITE_BUSY; + }else if( p->nRdLock>1 ){ + rc = SQLITE_BUSY; + } + p->nWrLock = 1; }else if( pThis->eLock<=SQLITE_LOCK_SHARED ){ + /* Upgrading to RESERVED or PENDING from SHARED. Fail if any other + ** client has a write-lock of any kind. */ if( p->nWrLock ){ rc = SQLITE_BUSY; }else{ From 2bb4bdc945eac5d6316d34911b54856b4923481a Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Thu, 10 Aug 2023 16:17:07 -0400 Subject: [PATCH 21/39] Enable FTS5 This requires linking the mathematical library, libm. --- sqlite3.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sqlite3.go b/sqlite3.go index b3711314..b428a2b7 100644 --- a/sqlite3.go +++ b/sqlite3.go @@ -16,6 +16,8 @@ package sqlite3 #cgo CFLAGS: -DHAVE_USLEEP=1 #cgo CFLAGS: -DSQLITE_ENABLE_FTS3 #cgo CFLAGS: -DSQLITE_ENABLE_FTS3_PARENTHESIS +#cgo CFLAGS: -DSQLITE_ENABLE_FTS5 +#cgo LDFLAGS: -lm #cgo CFLAGS: -DSQLITE_TRACE_SIZE_LIMIT=15 #cgo CFLAGS: -DSQLITE_OMIT_DEPRECATED #cgo CFLAGS: -DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1 From f6c614edd14e8ded7a57b892903ff065ffc4f3b7 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Thu, 25 Jan 2024 21:41:22 -0500 Subject: [PATCH 22/39] Remove the file --- .github/workflows/go.yaml | 115 -------------------------------------- 1 file changed, 115 deletions(-) delete mode 100644 .github/workflows/go.yaml diff --git a/.github/workflows/go.yaml b/.github/workflows/go.yaml deleted file mode 100644 index c96bf31a..00000000 --- a/.github/workflows/go.yaml +++ /dev/null @@ -1,115 +0,0 @@ -name: Go - -on: [push, pull_request] - -jobs: - - test: - name: Test - runs-on: ${{ matrix.os }} - defaults: - run: - shell: bash - - strategy: - matrix: - os: [ubuntu-latest, macos-latest] - go: ['1.19', '1.20', '1.21'] - fail-fast: false - env: - OS: ${{ matrix.os }} - GO: ${{ matrix.go }} - steps: - - if: startsWith(matrix.os, 'macos') - run: brew update - - - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go }} - - - name: Get Build Tools - run: | - GO111MODULE=on go install github.com/ory/go-acc@latest - - - name: Add $GOPATH/bin to $PATH - run: | - echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH" - - - uses: actions/checkout@v2 - - - name: 'Tags: default' - run: go-acc . -- -race -v -tags "" - - - name: 'Tags: libsqlite3' - run: go-acc . -- -race -v -tags "libsqlite3" - - - name: 'Tags: full' - run: go-acc . -- -race -v -tags "sqlite_allow_uri_authority sqlite_app_armor sqlite_column_metadata sqlite_foreign_keys sqlite_fts5 sqlite_icu sqlite_introspect sqlite_json sqlite_math_functions sqlite_os_trace sqlite_preupdate_hook sqlite_secure_delete sqlite_see sqlite_stat4 sqlite_trace sqlite_unlock_notify sqlite_userauth sqlite_vacuum_incr sqlite_vtable" - - - name: 'Tags: vacuum' - run: go-acc . -- -race -v -tags "sqlite_vacuum_full" - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v1 - with: - env_vars: OS,GO - file: coverage.txt - - test-windows: - name: Test for Windows - runs-on: windows-latest - defaults: - run: - shell: bash - - strategy: - matrix: - go: ['1.19', '1.20', '1.21'] - fail-fast: false - env: - OS: windows-latest - GO: ${{ matrix.go }} - steps: - - uses: msys2/setup-msys2@v2 - with: - update: true - install: mingw-w64-x86_64-toolchain mingw-w64-x86_64-sqlite3 - msystem: MINGW64 - path-type: inherit - - - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go }} - - - name: Add $GOPATH/bin to $PATH - run: | - echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH" - shell: msys2 {0} - - - uses: actions/checkout@v2 - - - name: 'Tags: default' - run: go build -race -v -tags "" - shell: msys2 {0} - - - name: 'Tags: libsqlite3' - run: go build -race -v -tags "libsqlite3" - shell: msys2 {0} - - - name: 'Tags: full' - run: | - echo 'skip this test' - echo go build -race -v -tags "sqlite_allow_uri_authority sqlite_app_armor sqlite_column_metadata sqlite_foreign_keys sqlite_fts5 sqlite_icu sqlite_introspect sqlite_json sqlite_math_functions sqlite_preupdate_hook sqlite_secure_delete sqlite_see sqlite_stat4 sqlite_trace sqlite_unlock_notify sqlite_userauth sqlite_vacuum_incr sqlite_vtable" - shell: msys2 {0} - - - name: 'Tags: vacuum' - run: go build -race -v -tags "sqlite_vacuum_full" - shell: msys2 {0} - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2 - with: - env_vars: OS,GO - file: coverage.txt - -# based on: github.com/koron-go/_skeleton/.github/workflows/go.yml From 1fd986cc81137f408dd90bb52224453fb75060e6 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Tue, 11 Jun 2024 23:12:28 -0400 Subject: [PATCH 23/39] Support disabling checkpoint-on-close Because Go doesn't support calling varadic C functions, this required a wrapper. --- sqlite3.go | 16 +++++++++ sqlite3_test.go | 87 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/sqlite3.go b/sqlite3.go index 4ff905b5..9a8a6788 100644 --- a/sqlite3.go +++ b/sqlite3.go @@ -84,6 +84,12 @@ _sqlite3_bind_blob(sqlite3_stmt *stmt, int n, void *p, int np) { return sqlite3_bind_blob(stmt, n, p, np, SQLITE_TRANSIENT); } +static int +_sqlite3_db_config_no_ckpt_on_close(sqlite3 *db) { + int v; + return sqlite3_db_config(db, SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, 1, &v); +} + #include #include @@ -1898,6 +1904,16 @@ func (c *SQLiteConn) SetFileControlInt(dbName string, op int, arg int) error { return nil } +// DBConfigNoCkptOnClose disables checkpointing on database close. +// See http://sqlite.org/c3ref/db_config.html +func (c *SQLiteConn) DBConfigNoCkptOnClose() error { + rv := C._sqlite3_db_config_no_ckpt_on_close(c.db) + if rv != C.SQLITE_OK { + return c.lastError() + } + return nil +} + // Close the statement. func (s *SQLiteStmt) Close() error { s.mu.Lock() diff --git a/sqlite3_test.go b/sqlite3_test.go index 63c939d3..5e2c8ca9 100644 --- a/sqlite3_test.go +++ b/sqlite3_test.go @@ -10,6 +10,7 @@ package sqlite3 import ( "bytes" + "context" "database/sql" "database/sql/driver" "errors" @@ -1864,6 +1865,92 @@ func TestSetFileControlInt(t *testing.T) { }) } +func TestDBConfigNoCkptOnClose(t *testing.T) { + fname := TempFilename(t) + defer os.Remove(fname) + db, err := sql.Open("sqlite3", fname) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + // Enable WAL mode. + if _, err := db.Exec(`PRAGMA journal_mode = wal`); err != nil { + t.Fatal(err) + } + + // Write some data. + _, err = db.Exec("create table foo (department integer, profits integer)") + if err != nil { + t.Fatal("Failed to create table:", err) + } + + // Confirm WAL file exists. + if _, err := os.Stat(fname + "-wal"); err != nil { + t.Fatal("Expected WAL file to exist", err) + } + + // Close the database, and confirm WAL file is removed. + if err := db.Close(); err != nil { + t.Fatal("Failed to close database", err) + } + if _, err := os.Stat(fname + "-wal"); err == nil { + t.Fatal("Expected WAL file to be removed after close") + } + + // Now do it again, but with the DBConfig option set. + db, err = sql.Open("sqlite3", fname) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + // Insert a record, confirm a WAL file appears. + if _, err := db.Exec(`insert into foo values (1, 2)`); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(fname + "-wal"); err != nil { + t.Fatal("Expected WAL file to exist", err) + } + + // Disable checkpoint-on-close. + f := func(driverConn interface{}) error { + c := driverConn.(*SQLiteConn) + return c.DBConfigNoCkptOnClose() + } + conn, err := db.Conn(context.Background()) + if err != nil { + t.Fatal(err) + } + if err := conn.Raw(f); err != nil { + t.Fatal(err) + } + + // Read the SQLite file into a byte slice for comparison later. + bufPre, err := os.ReadFile(fname) + if err != nil { + t.Fatal(err) + } + + // Close the database, and confirm WAL file is still present. + if err := db.Close(); err != nil { + t.Fatal("Failed to close database", err) + } + if _, err := os.Stat(fname + "-wal"); err != nil { + t.Fatal("Expected WAL file to be present after close", err) + } + + // Confirm the SQLite file is the same as before since no checkpoint + // was performed. + bufPost, err := os.ReadFile(fname) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(bufPre, bufPost) { + t.Fatal("Expected SQLite file to be unchanged after close") + } +} + func TestNonColumnString(t *testing.T) { db, err := sql.Open("sqlite3", ":memory:") if err != nil { From ffc020209b0f2ed46ab9b078707c4623006eceda Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Thu, 8 Aug 2024 17:50:28 -0400 Subject: [PATCH 24/39] Use a nil pointer is entry is "" --- sqlite3_load_extension.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sqlite3_load_extension.go b/sqlite3_load_extension.go index 03cbc8b6..eb72fd41 100644 --- a/sqlite3_load_extension.go +++ b/sqlite3_load_extension.go @@ -50,7 +50,11 @@ func (c *SQLiteConn) LoadExtension(lib string, entry string) error { return errors.New(C.GoString(C.sqlite3_errmsg(c.db))) } - if err := c.loadExtension(lib, &entry); err != nil { + var ptr *string + if entry != "" { + ptr = &entry + } + if err := c.loadExtension(lib, ptr); err != nil { C.sqlite3_enable_load_extension(c.db, 0) return err } From 241bc715404108f6fe44645e54dcc22106256b74 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Mon, 9 Sep 2024 15:06:16 -0400 Subject: [PATCH 25/39] Enable Geopoly --- sqlite3.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sqlite3.go b/sqlite3.go index 9a8a6788..fb0ee54c 100644 --- a/sqlite3.go +++ b/sqlite3.go @@ -26,6 +26,7 @@ package sqlite3 #cgo CFLAGS: -DSQLITE_ENABLE_JSON1 #cgo CFLAGS: -DSQLITE_ENABLE_DBSTAT_VTAB #cgo CFLAGS: -DSQLITE_MEMDB_DEFAULT_MAXSIZE=2147483648 +#cgo CFLAGS: -DSQLITE_ENABLE_GEOPOLY=1 #cgo CFLAGS: -Wno-deprecated-declarations #cgo openbsd CFLAGS: -I/usr/local/include #cgo openbsd LDFLAGS: -L/usr/local/lib From 30e901811725cacb885a301024f828a802f85136 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Sat, 11 Jan 2025 10:21:37 -0500 Subject: [PATCH 26/39] Always compile and test preupdate hook --- sqlite3_opt_preupdate_hook.go | 3 --- sqlite3_opt_preupdate_hook_test.go | 3 --- sqlite3_opt_preupdate_omit.go | 22 ---------------------- 3 files changed, 28 deletions(-) delete mode 100644 sqlite3_opt_preupdate_omit.go diff --git a/sqlite3_opt_preupdate_hook.go b/sqlite3_opt_preupdate_hook.go index 8cce278f..5997aa37 100644 --- a/sqlite3_opt_preupdate_hook.go +++ b/sqlite3_opt_preupdate_hook.go @@ -4,9 +4,6 @@ // Use of this source code is governed by an MIT-style // license that can be found in the LICENSE file. -//go:build sqlite_preupdate_hook -// +build sqlite_preupdate_hook - package sqlite3 /* diff --git a/sqlite3_opt_preupdate_hook_test.go b/sqlite3_opt_preupdate_hook_test.go index 48926028..6386c750 100644 --- a/sqlite3_opt_preupdate_hook_test.go +++ b/sqlite3_opt_preupdate_hook_test.go @@ -4,9 +4,6 @@ // Use of this source code is governed by an MIT-style // license that can be found in the LICENSE file. -//go:build sqlite_preupdate_hook -// +build sqlite_preupdate_hook - package sqlite3 import ( diff --git a/sqlite3_opt_preupdate_omit.go b/sqlite3_opt_preupdate_omit.go deleted file mode 100644 index f60da6c1..00000000 --- a/sqlite3_opt_preupdate_omit.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (C) 2019 G.J.R. Timmer . -// Copyright (C) 2018 segment.com -// -// Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file. - -//go:build !sqlite_preupdate_hook && cgo -// +build !sqlite_preupdate_hook,cgo - -package sqlite3 - -// RegisterPreUpdateHook sets the pre-update hook for a connection. -// -// The callback is passed a SQLitePreUpdateData struct with the data for -// the update, as well as methods for fetching copies of impacted data. -// -// If there is an existing preupdate hook for this connection, it will be -// removed. If callback is nil the existing hook (if any) will be removed -// without creating a new one. -func (c *SQLiteConn) RegisterPreUpdateHook(callback func(SQLitePreUpdateData)) { - // NOOP -} From ace797763fd4cd97fc8625ac43d65546336a4613 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Sat, 11 Jan 2025 11:20:27 -0500 Subject: [PATCH 27/39] Add stubbed out support for Sessions --- sqlite3_session.go | 74 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 sqlite3_session.go diff --git a/sqlite3_session.go b/sqlite3_session.go new file mode 100644 index 00000000..ca188458 --- /dev/null +++ b/sqlite3_session.go @@ -0,0 +1,74 @@ +//go:build !sqlite_omit_session +// +build !sqlite_omit_session + +package sqlite3 + +/* +#cgo CFLAGS: -DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_PREUPDATE_HOOK +#cgo LDFLAGS: -lm +*/ + +/* +#ifndef USE_LIBSQLITE3 +#include "sqlite3-binding.h" +#else +#include +#endif +#include +*/ +import "C" + +// CreateSession creates a new session object. +func (c *SQLiteConn) CreateSession() error { + return nil +} + +// AttachSession attaches a session object to a set of tables. +func (c *SQLiteConn) AttachSession() error { + return nil +} + +// DetachSession deletes a session object. +func (c *SQLiteConn) DeleteSession() error { + return nil +} + +// SessionChangeset generates a changeset from a session object. +func (c *SQLiteConn) Changeset() error { + return nil +} + +// ChangesetStart is called to create and initialize an iterator +// to iterate through the contents of a changeset. Initially, the +// iterator points to no element at all +func (c *SQLiteConn) ChangesetStart() error { + return nil +} + +// ChangesetNext moves a Changeset iterator to the next change in the +// changeset. +func (c *SQLiteConn) ChangesetNext() error { + return nil +} + +// ChangesetOp retuns the type of change (INSERT, UPDATE or DELETE) +// that the iterator points to +func (c *SQLiteConn) ChangesetOp() error { + return nil +} + +// ChangesetOld may be used to obtain the old.* values within the change payload. +func (c *SQLiteConn) ChangesetOld() error { + return nil +} + +// ChangesetNew may be used to obtain the new.* values within the change payload. +func (c *SQLiteConn) ChangesetNew() error { + return nil +} + +// ChangesetFinalize is called to delete a changeste iterator. +func (c *SQLiteConn) ChangesetFinalize() error { + return nil +} + From cdf5f66152a559d2173324d3d7e3a6d809e430ad Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Sat, 11 Jan 2025 13:18:57 -0500 Subject: [PATCH 28/39] First version of implementation --- sqlite3.go | 2 + sqlite3_session.go | 155 ++++++++++++++++++++++++++++++++++++--------- 2 files changed, 126 insertions(+), 31 deletions(-) diff --git a/sqlite3.go b/sqlite3.go index a01b2ecd..44edf7d0 100644 --- a/sqlite3.go +++ b/sqlite3.go @@ -27,6 +27,8 @@ package sqlite3 #cgo CFLAGS: -DSQLITE_ENABLE_DBSTAT_VTAB #cgo CFLAGS: -DSQLITE_MEMDB_DEFAULT_MAXSIZE=2147483648 #cgo CFLAGS: -DSQLITE_ENABLE_GEOPOLY=1 +#cgo CFLAGS: -DSQLITE_ENABLE_SESSION +#cgo CFLAGS: -DSQLITE_ENABLE_PREUPDATE_HOOK #cgo CFLAGS: -Wno-deprecated-declarations #cgo openbsd CFLAGS: -I/usr/local/include #cgo openbsd LDFLAGS: -L/usr/local/lib diff --git a/sqlite3_session.go b/sqlite3_session.go index ca188458..d136899d 100644 --- a/sqlite3_session.go +++ b/sqlite3_session.go @@ -15,60 +15,153 @@ package sqlite3 #include #endif #include +#include */ import "C" +import ( + "fmt" + "unsafe" +) + +type Session struct { + session *C.sqlite3_session +} + +type Changeset struct { + changeset *C.sqlite3_changeset_iter +} // CreateSession creates a new session object. -func (c *SQLiteConn) CreateSession() error { - return nil +func (c *SQLiteConn) CreateSession(dbName string) (*Session, error) { + cDbName := C.CString(dbName) + defer C.free(unsafe.Pointer(cDbName)) + + var session *C.sqlite3_session + rc := C.sqlite3session_create(c.db, cDbName, &session) + if rc != C.SQLITE_OK { + return nil, fmt.Errorf("sqlite3session_create: %s", C.GoString(C.sqlite3_errstr(rc))) + } + return &Session{session: session}, nil } -// AttachSession attaches a session object to a set of tables. -func (c *SQLiteConn) AttachSession() error { +// AttachSession attaches a session object to a table or all tables. +func (s *Session) AttachSession(tableName string) error { + var cTableName *C.char + if tableName != "" { + cTableName = C.CString(tableName) + defer C.free(unsafe.Pointer(cTableName)) + } + + rc := C.sqlite3session_attach(s.session, cTableName) + if rc != C.SQLITE_OK { + return fmt.Errorf("sqlite3session_attach: %s", C.GoString(C.sqlite3_errstr(rc))) + } return nil } -// DetachSession deletes a session object. -func (c *SQLiteConn) DeleteSession() error { +// Delete deletes a session object. +func (s *Session) DeleteSession() error { + if s.session != nil { + // Call sqlite3session_delete to free the session object + C.sqlite3session_delete(s.session) + s.session = nil // Set session to nil to avoid double deletion + } return nil } -// SessionChangeset generates a changeset from a session object. -func (c *SQLiteConn) Changeset() error { - return nil +type ChangesetIter struct { + iter *C.sqlite3_changeset_iter } -// ChangesetStart is called to create and initialize an iterator -// to iterate through the contents of a changeset. Initially, the -// iterator points to no element at all -func (c *SQLiteConn) ChangesetStart() error { - return nil +// Changeset generates a changeset from a session object. +func (s *Session) Changeset() ([]byte, error) { + var nChangeset C.int + var pChangeset unsafe.Pointer + + // Call sqlite3session_changeset + rc := C.sqlite3session_changeset(s.session, &nChangeset, &pChangeset) + if rc != C.SQLITE_OK { + return nil, fmt.Errorf("sqlite3session_changeset: %s", C.GoString(C.sqlite3_errstr(rc))) + } + defer C.sqlite3_free(pChangeset) // Free the changeset buffer after use + + // Convert the C buffer to a Go byte slice + changeset := C.GoBytes(pChangeset, nChangeset) + return changeset, nil } -// ChangesetNext moves a Changeset iterator to the next change in the -// changeset. -func (c *SQLiteConn) ChangesetNext() error { - return nil +// ChangesetStart creates and initializes a changeset iterator. +func ChangesetStart(changeset []byte) (*ChangesetIter, error) { + var iter *C.sqlite3_changeset_iter + + // Call sqlite3changeset_start + rc := C.sqlite3changeset_start(&iter, C.int(len(changeset)), unsafe.Pointer(&changeset[0])) + if rc != C.SQLITE_OK { + return nil, fmt.Errorf("sqlite3changeset_start: %s", C.GoString(C.sqlite3_errstr(rc))) + } + + return &ChangesetIter{iter: iter}, nil } -// ChangesetOp retuns the type of change (INSERT, UPDATE or DELETE) -// that the iterator points to -func (c *SQLiteConn) ChangesetOp() error { - return nil +// ChangesetNext moves the changeset iterator to the next change. +func (ci *ChangesetIter) ChangesetNext() (bool, error) { + rc := C.sqlite3changeset_next(ci.iter) + if rc == C.SQLITE_DONE { + return false, nil // No more changes + } + if rc != C.SQLITE_OK { + return false, fmt.Errorf("sqlite3changeset_next: %s", C.GoString(C.sqlite3_errstr(rc))) + } + return true, nil } -// ChangesetOld may be used to obtain the old.* values within the change payload. -func (c *SQLiteConn) ChangesetOld() error { - return nil +// ChangesetOp returns the type of change (INSERT, UPDATE, or DELETE) that the iterator points to. +func (ci *ChangesetIter) ChangesetOp() (string, int, int, bool, error) { + var tableName *C.char + var nCol C.int + var op C.int + var indirect C.int + + rc := C.sqlite3changeset_op(ci.iter, &tableName, &nCol, &op, &indirect) + if rc != C.SQLITE_OK { + return "", 0, 0, false, fmt.Errorf("sqlite3changeset_op: %s", C.GoString(C.sqlite3_errstr(rc))) + } + + return C.GoString(tableName), int(nCol), int(op), indirect != 0, nil } -// ChangesetNew may be used to obtain the new.* values within the change payload. -func (c *SQLiteConn) ChangesetNew() error { - return nil +// ChangesetOld retrieves the old value for the specified column in the change payload. +func (ci *ChangesetIter) ChangesetOld(column int) (*C.sqlite3_value, error) { + var value *C.sqlite3_value + + rc := C.sqlite3changeset_old(ci.iter, C.int(column), &value) + if rc != C.SQLITE_OK { + return nil, fmt.Errorf("sqlite3changeset_old: %s", C.GoString(C.sqlite3_errstr(rc))) + } + + return value, nil } -// ChangesetFinalize is called to delete a changeste iterator. -func (c *SQLiteConn) ChangesetFinalize() error { - return nil +// ChangesetNew retrieves the new value for the specified column in the change payload. +func (ci *ChangesetIter) ChangesetNew(column int) (*C.sqlite3_value, error) { + var value *C.sqlite3_value + + rc := C.sqlite3changeset_new(ci.iter, C.int(column), &value) + if rc != C.SQLITE_OK { + return nil, fmt.Errorf("sqlite3changeset_new: %s", C.GoString(C.sqlite3_errstr(rc))) + } + + return value, nil } +// ChangesetFinalize deletes a changeset iterator. +func (ci *ChangesetIter) ChangesetFinalize() error { + if ci.iter != nil { + rc := C.sqlite3changeset_finalize(ci.iter) + ci.iter = nil // Prevent double finalization + if rc != C.SQLITE_OK { + return fmt.Errorf("sqlite3changeset_finalize: %s", C.GoString(C.sqlite3_errstr(rc))) + } + } + return nil +} From 2c404eef14583283240f6836871efe8c57790cfc Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Sat, 11 Jan 2025 13:19:42 -0500 Subject: [PATCH 29/39] Remove unneeded type --- sqlite3_session.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sqlite3_session.go b/sqlite3_session.go index d136899d..31703daa 100644 --- a/sqlite3_session.go +++ b/sqlite3_session.go @@ -27,10 +27,6 @@ type Session struct { session *C.sqlite3_session } -type Changeset struct { - changeset *C.sqlite3_changeset_iter -} - // CreateSession creates a new session object. func (c *SQLiteConn) CreateSession(dbName string) (*Session, error) { cDbName := C.CString(dbName) From 2a0ca73ee407dccf02b378ea9a982aaadf68ac8a Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Sat, 11 Jan 2025 17:01:16 -0500 Subject: [PATCH 30/39] More OK compilation --- sqlite3_session.go | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/sqlite3_session.go b/sqlite3_session.go index 31703daa..5271775d 100644 --- a/sqlite3_session.go +++ b/sqlite3_session.go @@ -65,12 +65,16 @@ func (s *Session) DeleteSession() error { return nil } -type ChangesetIter struct { +type Changeset struct { + cs []byte +} + +type ChangesetIterator struct { iter *C.sqlite3_changeset_iter } // Changeset generates a changeset from a session object. -func (s *Session) Changeset() ([]byte, error) { +func (s *Session) Changeset() (*Changeset, error) { var nChangeset C.int var pChangeset unsafe.Pointer @@ -81,13 +85,13 @@ func (s *Session) Changeset() ([]byte, error) { } defer C.sqlite3_free(pChangeset) // Free the changeset buffer after use - // Convert the C buffer to a Go byte slice + // copy the changeset buffer to a Go byte slice, because cgo can nuke its memory at any time changeset := C.GoBytes(pChangeset, nChangeset) - return changeset, nil + return &Changeset{cs: changeset}, nil } // ChangesetStart creates and initializes a changeset iterator. -func ChangesetStart(changeset []byte) (*ChangesetIter, error) { +func ChangesetStart(changeset []byte) (*ChangesetIterator, error) { var iter *C.sqlite3_changeset_iter // Call sqlite3changeset_start @@ -96,11 +100,15 @@ func ChangesetStart(changeset []byte) (*ChangesetIter, error) { return nil, fmt.Errorf("sqlite3changeset_start: %s", C.GoString(C.sqlite3_errstr(rc))) } - return &ChangesetIter{iter: iter}, nil + return &ChangesetIterator{iter: iter}, nil +} + +func (cs *Changeset) Start() (*ChangesetIterator, error) { + return ChangesetStart(cs.cs) } // ChangesetNext moves the changeset iterator to the next change. -func (ci *ChangesetIter) ChangesetNext() (bool, error) { +func (ci *ChangesetIterator) ChangesetNext() (bool, error) { rc := C.sqlite3changeset_next(ci.iter) if rc == C.SQLITE_DONE { return false, nil // No more changes @@ -112,7 +120,7 @@ func (ci *ChangesetIter) ChangesetNext() (bool, error) { } // ChangesetOp returns the type of change (INSERT, UPDATE, or DELETE) that the iterator points to. -func (ci *ChangesetIter) ChangesetOp() (string, int, int, bool, error) { +func (ci *ChangesetIterator) ChangesetOp() (string, int, int, bool, error) { var tableName *C.char var nCol C.int var op C.int @@ -127,7 +135,7 @@ func (ci *ChangesetIter) ChangesetOp() (string, int, int, bool, error) { } // ChangesetOld retrieves the old value for the specified column in the change payload. -func (ci *ChangesetIter) ChangesetOld(column int) (*C.sqlite3_value, error) { +func (ci *ChangesetIterator) ChangesetOld(column int) (*C.sqlite3_value, error) { var value *C.sqlite3_value rc := C.sqlite3changeset_old(ci.iter, C.int(column), &value) @@ -139,7 +147,7 @@ func (ci *ChangesetIter) ChangesetOld(column int) (*C.sqlite3_value, error) { } // ChangesetNew retrieves the new value for the specified column in the change payload. -func (ci *ChangesetIter) ChangesetNew(column int) (*C.sqlite3_value, error) { +func (ci *ChangesetIterator) ChangesetNew(column int) (*C.sqlite3_value, error) { var value *C.sqlite3_value rc := C.sqlite3changeset_new(ci.iter, C.int(column), &value) @@ -151,7 +159,7 @@ func (ci *ChangesetIter) ChangesetNew(column int) (*C.sqlite3_value, error) { } // ChangesetFinalize deletes a changeset iterator. -func (ci *ChangesetIter) ChangesetFinalize() error { +func (ci *ChangesetIterator) ChangesetFinalize() error { if ci.iter != nil { rc := C.sqlite3changeset_finalize(ci.iter) ci.iter = nil // Prevent double finalization From d7ea36b7fc4e059ab56ae5331046a1615ce5f15d Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Sat, 11 Jan 2025 18:18:50 -0500 Subject: [PATCH 31/39] Add first Session unit test --- sqlite3_session_test.go | 69 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 sqlite3_session_test.go diff --git a/sqlite3_session_test.go b/sqlite3_session_test.go new file mode 100644 index 00000000..9fc25a44 --- /dev/null +++ b/sqlite3_session_test.go @@ -0,0 +1,69 @@ +package sqlite3 + +import ( + "context" + "database/sql" + "testing" +) + +func Test_EmptyChangeset(t *testing.T) { + // Open a new in-memory SQLite database + db, err := sql.Open("sqlite3", ":memory:") + if err != nil { + t.Fatal("Failed to open", err) + } + err = db.Ping() + if err != nil { + t.Fatal("Failed to ping", err) + } + defer db.Close() + + _, err = db.Exec(`CREATE TABLE test_table (id INTEGER PRIMARY KEY, value TEXT);`) + if err != nil { + t.Fatalf("Failed to create table: %v", err) + } + + conn, err := db.Conn(context.Background()) + if err != nil { + t.Fatal("Failed to get connection to source database:", err) + } + defer conn.Close() + + var session *Session + if err := conn.Raw(func(raw any) error { + var err error + session, err = raw.(*SQLiteConn).CreateSession("main") + return err + }); err != nil { + t.Fatal("Failed to serialize source database:", err) + } + defer func() { + if err := session.DeleteSession(); err != nil { + t.Errorf("Failed to delete session: %v", err) + } + }() + + err = session.AttachSession("test_table") + if err != nil { + t.Fatalf("Failed to attach session to table: %v", err) + } + + changeset, err := session.Changeset() + if err != nil { + t.Fatalf("Failed to generate changeset: %v", err) + } + + iter, err := NewChangesetIterator(changeset) + if err != nil { + t.Fatalf("Failed to create changeset iterator: %v", err) + } + if b, err := iter.Next(); err != nil { + t.Fatalf("Failed to get next changeset: %v", err) + } else if b { + t.Fatalf("changeset contains changes: %v", b) + } + + if err := iter.Finalize(); err != nil { + t.Fatalf("Failed to finalize changeset iterator: %v", err) + } +} From d79d90bee4d288b97bda21c8403c670ae354b569 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Sun, 12 Jan 2025 11:17:19 -0500 Subject: [PATCH 32/39] Basically working, but type failures --- sqlite3_session.go | 147 ++++++++++++++++++++++++++-------------- sqlite3_session_test.go | 108 ++++++++++++++++++++++++++++- 2 files changed, 202 insertions(+), 53 deletions(-) diff --git a/sqlite3_session.go b/sqlite3_session.go index 5271775d..ff1a8309 100644 --- a/sqlite3_session.go +++ b/sqlite3_session.go @@ -58,111 +58,156 @@ func (s *Session) AttachSession(tableName string) error { // Delete deletes a session object. func (s *Session) DeleteSession() error { if s.session != nil { - // Call sqlite3session_delete to free the session object C.sqlite3session_delete(s.session) - s.session = nil // Set session to nil to avoid double deletion + s.session = nil } return nil } +// Changeset represents a changeset object. type Changeset struct { - cs []byte + b []byte } -type ChangesetIterator struct { - iter *C.sqlite3_changeset_iter -} - -// Changeset generates a changeset from a session object. -func (s *Session) Changeset() (*Changeset, error) { +// NewChangeset returns a changeset from a session object. +func NewChangeset(s *Session) (*Changeset, error) { var nChangeset C.int var pChangeset unsafe.Pointer - // Call sqlite3session_changeset rc := C.sqlite3session_changeset(s.session, &nChangeset, &pChangeset) if rc != C.SQLITE_OK { return nil, fmt.Errorf("sqlite3session_changeset: %s", C.GoString(C.sqlite3_errstr(rc))) } - defer C.sqlite3_free(pChangeset) // Free the changeset buffer after use + defer C.sqlite3_free(pChangeset) - // copy the changeset buffer to a Go byte slice, because cgo can nuke its memory at any time + // Copy the changeset buffer to a Go byte slice, because cgo + // does not support Go slices with C memory. changeset := C.GoBytes(pChangeset, nChangeset) - return &Changeset{cs: changeset}, nil + return &Changeset{b: changeset}, nil } -// ChangesetStart creates and initializes a changeset iterator. -func ChangesetStart(changeset []byte) (*ChangesetIterator, error) { - var iter *C.sqlite3_changeset_iter +// ChangesetIterator represents a changeset iterator object. +type ChangesetIterator struct { + iter *C.sqlite3_changeset_iter +} - // Call sqlite3changeset_start - rc := C.sqlite3changeset_start(&iter, C.int(len(changeset)), unsafe.Pointer(&changeset[0])) +// NewChangesetIterator creates a new changeset iterator object. +func NewChangesetIterator(cs *Changeset) (*ChangesetIterator, error) { + var iter *C.sqlite3_changeset_iter + ptr := unsafe.Pointer(nil) + if len(cs.b) > 0 { + ptr = unsafe.Pointer(&cs.b[0]) + } + rc := C.sqlite3changeset_start(&iter, C.int(len(cs.b)), ptr) if rc != C.SQLITE_OK { return nil, fmt.Errorf("sqlite3changeset_start: %s", C.GoString(C.sqlite3_errstr(rc))) } - return &ChangesetIterator{iter: iter}, nil } -func (cs *Changeset) Start() (*ChangesetIterator, error) { - return ChangesetStart(cs.cs) -} - -// ChangesetNext moves the changeset iterator to the next change. -func (ci *ChangesetIterator) ChangesetNext() (bool, error) { +// Next moves the changeset iterator to the next change. +func (ci *ChangesetIterator) Next() (bool, error) { rc := C.sqlite3changeset_next(ci.iter) if rc == C.SQLITE_DONE { return false, nil // No more changes } - if rc != C.SQLITE_OK { + if rc != C.SQLITE_ROW { return false, fmt.Errorf("sqlite3changeset_next: %s", C.GoString(C.sqlite3_errstr(rc))) } return true, nil } -// ChangesetOp returns the type of change (INSERT, UPDATE, or DELETE) that the iterator points to. -func (ci *ChangesetIterator) ChangesetOp() (string, int, int, bool, error) { +// Op returns the current Operation from a Changeset Iterator +func (ci *ChangesetIterator) Op() (tblName string, numCol int, oper int, indirect bool, err error) { var tableName *C.char var nCol C.int var op C.int - var indirect C.int + var ind C.int - rc := C.sqlite3changeset_op(ci.iter, &tableName, &nCol, &op, &indirect) + rc := C.sqlite3changeset_op(ci.iter, &tableName, &nCol, &op, &ind) if rc != C.SQLITE_OK { return "", 0, 0, false, fmt.Errorf("sqlite3changeset_op: %s", C.GoString(C.sqlite3_errstr(rc))) } - - return C.GoString(tableName), int(nCol), int(op), indirect != 0, nil + return C.GoString(tableName), int(nCol), int(op), ind != 0, nil } -// ChangesetOld retrieves the old value for the specified column in the change payload. -func (ci *ChangesetIterator) ChangesetOld(column int) (*C.sqlite3_value, error) { - var value *C.sqlite3_value +// Old retrieves the old value for the specified column in the change payload. +func (ci *ChangesetIterator) Old(dest []any) error { + for i := 0; i < len(dest); i++ { + var val *C.sqlite3_value + var src any - rc := C.sqlite3changeset_old(ci.iter, C.int(column), &value) - if rc != C.SQLITE_OK { - return nil, fmt.Errorf("sqlite3changeset_old: %s", C.GoString(C.sqlite3_errstr(rc))) - } + rc := C.sqlite3changeset_old(ci.iter, C.int(i), &val) + if rc != C.SQLITE_OK { + return fmt.Errorf("sqlite3changeset_old: %s", C.GoString(C.sqlite3_errstr(rc))) + } + + switch C.sqlite3_value_type(val) { + case C.SQLITE_INTEGER: + src = int64(C.sqlite3_value_int64(val)) + case C.SQLITE_FLOAT: + src = float64(C.sqlite3_value_double(val)) + case C.SQLITE_BLOB: + len := C.sqlite3_value_bytes(val) + blobptr := C.sqlite3_value_blob(val) + src = C.GoBytes(blobptr, len) + case C.SQLITE_TEXT: + len := C.sqlite3_value_bytes(val) + cstrptr := unsafe.Pointer(C.sqlite3_value_text(val)) + src = C.GoBytes(cstrptr, len) + case C.SQLITE_NULL: + src = nil + } - return value, nil + err := convertAssign(&dest[i], src) + if err != nil { + return err + } + } + return nil } -// ChangesetNew retrieves the new value for the specified column in the change payload. -func (ci *ChangesetIterator) ChangesetNew(column int) (*C.sqlite3_value, error) { - var value *C.sqlite3_value +// New retrieves the new value for the specified column in the change payload. +func (ci *ChangesetIterator) New(dest []any) error { + for i := 0; i < len(dest); i++ { + var val *C.sqlite3_value + var src any - rc := C.sqlite3changeset_new(ci.iter, C.int(column), &value) - if rc != C.SQLITE_OK { - return nil, fmt.Errorf("sqlite3changeset_new: %s", C.GoString(C.sqlite3_errstr(rc))) - } + rc := C.sqlite3changeset_new(ci.iter, C.int(i), &val) + if rc != C.SQLITE_OK { + return fmt.Errorf("sqlite3changeset_new: %s", C.GoString(C.sqlite3_errstr(rc))) + } + + switch C.sqlite3_value_type(val) { + case C.SQLITE_INTEGER: + src = int64(C.sqlite3_value_int64(val)) + case C.SQLITE_FLOAT: + src = float64(C.sqlite3_value_double(val)) + case C.SQLITE_BLOB: + len := C.sqlite3_value_bytes(val) + blobptr := C.sqlite3_value_blob(val) + src = C.GoBytes(blobptr, len) + case C.SQLITE_TEXT: + len := C.sqlite3_value_bytes(val) + cstrptr := unsafe.Pointer(C.sqlite3_value_text(val)) + src = C.GoBytes(cstrptr, len) + case C.SQLITE_NULL: + src = nil + } - return value, nil + err := convertAssign(&dest[i], src) + if err != nil { + return err + } + } + return nil } -// ChangesetFinalize deletes a changeset iterator. -func (ci *ChangesetIterator) ChangesetFinalize() error { +// Finalize deletes a changeset iterator. +func (ci *ChangesetIterator) Finalize() error { if ci.iter != nil { rc := C.sqlite3changeset_finalize(ci.iter) - ci.iter = nil // Prevent double finalization + ci.iter = nil if rc != C.SQLITE_OK { return fmt.Errorf("sqlite3changeset_finalize: %s", C.GoString(C.sqlite3_errstr(rc))) } diff --git a/sqlite3_session_test.go b/sqlite3_session_test.go index 9fc25a44..0b5cb476 100644 --- a/sqlite3_session_test.go +++ b/sqlite3_session_test.go @@ -7,7 +7,6 @@ import ( ) func Test_EmptyChangeset(t *testing.T) { - // Open a new in-memory SQLite database db, err := sql.Open("sqlite3", ":memory:") if err != nil { t.Fatal("Failed to open", err) @@ -48,7 +47,7 @@ func Test_EmptyChangeset(t *testing.T) { t.Fatalf("Failed to attach session to table: %v", err) } - changeset, err := session.Changeset() + changeset, err := NewChangeset(session) if err != nil { t.Fatalf("Failed to generate changeset: %v", err) } @@ -67,3 +66,108 @@ func Test_EmptyChangeset(t *testing.T) { t.Fatalf("Failed to finalize changeset iterator: %v", err) } } + +func Test_Changeset_OneRow(t *testing.T) { + db, err := sql.Open("sqlite3", ":memory:") + if err != nil { + t.Fatal("Failed to open", err) + } + err = db.Ping() + if err != nil { + t.Fatal("Failed to ping", err) + } + defer db.Close() + + ctx := context.Background() + conn, err := db.Conn(ctx) + if err != nil { + t.Fatal("Failed to get connection to database:", err) + } + defer conn.Close() + + _, err = conn.ExecContext(ctx, `CREATE TABLE test_table (id INTEGER PRIMARY KEY, value TEXT)`) + if err != nil { + t.Fatalf("Failed to create table: %v", err) + } + + var session *Session + if err := conn.Raw(func(raw any) error { + var err error + session, err = raw.(*SQLiteConn).CreateSession("main") + return err + }); err != nil { + t.Fatal("Failed to create session:", err) + } + defer func() { + if err := session.DeleteSession(); err != nil { + t.Errorf("Failed to delete session: %v", err) + } + }() + + err = session.AttachSession("test_table") + if err != nil { + t.Fatalf("Failed to attach session to table: %v", err) + } + + _, err = conn.ExecContext(ctx, `INSERT INTO test_table (value) VALUES ('test')`) + if err != nil { + t.Fatalf("Failed to insert row: %v", err) + } + + changeset, err := NewChangeset(session) + if err != nil { + t.Fatalf("Failed to generate changeset: %v", err) + } + + iter, err := NewChangesetIterator(changeset) + if err != nil { + t.Fatalf("Failed to create changeset iterator: %v", err) + } + if b, err := iter.Next(); err != nil { + t.Fatalf("Failed to get next changeset: %v", err) + } else if !b { + t.Fatalf("changeset does not contain changes: %v", b) + } + + tblName, nCol, op, indirect, err := iter.Op() + if err != nil { + t.Fatalf("Failed to get changeset operation: %v", err) + } + if tblName != "test_table" { + t.Fatalf("Expected table name 'test_table', got '%s'", tblName) + } + if nCol != 2 { + t.Fatalf("Expected 2 columns, got %d", nCol) + } + if op != SQLITE_INSERT { + t.Fatalf("Expected operation 1, got %d", op) + } + if indirect { + t.Fatalf("Expected indirect false, got true") + } + + dest := make([]any, nCol) + if err := iter.New(dest); err != nil { + t.Fatalf("Failed to get new row: %v", err) + } + if v, ok := dest[0].(int64); !ok { + t.Fatalf("Expected int64, got %T", dest[0]) + } else if v != 1 { + t.Fatalf("Expected 1, got %d", v) + } + if v, ok := dest[1].(string); !ok { + t.Fatalf("Expected string, got %T", dest[1]) + } else if v != "test" { + t.Fatalf("Expected test, got %s", v) + } + + if b, err := iter.Next(); err != nil { + t.Fatalf("Failed to get next changeset: %v", err) + } else if b { + t.Fatalf("changeset contains more changes: %v", b) + } + + if err := iter.Finalize(); err != nil { + t.Fatalf("Failed to finalize changeset iterator: %v", err) + } +} From 6380ea084343c271b893ad6c042bed12e639e5c6 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Sun, 12 Jan 2025 11:29:57 -0500 Subject: [PATCH 33/39] Update config.yml --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cd7b0757..2390d6b2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ jobs: test: docker: # specify the version - - image: cimg/go:1.18.0 + - image: cimg/go:1.23.0 # Specify service dependencies here if necessary # CircleCI maintains a library of pre-built images @@ -25,7 +25,7 @@ jobs: - run: go version - run: go get -v -t -d ./... - run: go vet - - run: go test -v ./... + - run: go test -v -failfast ./... workflows: version: 2 From 9a6786b9b8bbdb5c9178f13e4c76cdb2c7cbeab7 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Sun, 12 Jan 2025 15:59:48 -0500 Subject: [PATCH 34/39] Refactor --- sqlite3_session.go | 86 ++++++++++++++++------------------------------ 1 file changed, 30 insertions(+), 56 deletions(-) diff --git a/sqlite3_session.go b/sqlite3_session.go index ff1a8309..ea0d2ee4 100644 --- a/sqlite3_session.go +++ b/sqlite3_session.go @@ -133,83 +133,57 @@ func (ci *ChangesetIterator) Op() (tblName string, numCol int, oper int, indirec // Old retrieves the old value for the specified column in the change payload. func (ci *ChangesetIterator) Old(dest []any) error { - for i := 0; i < len(dest); i++ { - var val *C.sqlite3_value - var src any - - rc := C.sqlite3changeset_old(ci.iter, C.int(i), &val) - if rc != C.SQLITE_OK { - return fmt.Errorf("sqlite3changeset_old: %s", C.GoString(C.sqlite3_errstr(rc))) - } + return ci.row(dest, true) +} - switch C.sqlite3_value_type(val) { - case C.SQLITE_INTEGER: - src = int64(C.sqlite3_value_int64(val)) - case C.SQLITE_FLOAT: - src = float64(C.sqlite3_value_double(val)) - case C.SQLITE_BLOB: - len := C.sqlite3_value_bytes(val) - blobptr := C.sqlite3_value_blob(val) - src = C.GoBytes(blobptr, len) - case C.SQLITE_TEXT: - len := C.sqlite3_value_bytes(val) - cstrptr := unsafe.Pointer(C.sqlite3_value_text(val)) - src = C.GoBytes(cstrptr, len) - case C.SQLITE_NULL: - src = nil - } +// New retrieves the new value for the specified column in the change payload. +func (ci *ChangesetIterator) New(dest []any) error { + return ci.row(dest, false) +} - err := convertAssign(&dest[i], src) - if err != nil { - return err +// Finalize deletes a changeset iterator. +func (ci *ChangesetIterator) Finalize() error { + if ci.iter != nil { + rc := C.sqlite3changeset_finalize(ci.iter) + ci.iter = nil + if rc != C.SQLITE_OK { + return fmt.Errorf("sqlite3changeset_finalize: %s", C.GoString(C.sqlite3_errstr(rc))) } } return nil } // New retrieves the new value for the specified column in the change payload. -func (ci *ChangesetIterator) New(dest []any) error { +func (ci *ChangesetIterator) row(dest []any, old bool) error { + var val *C.sqlite3_value + var rc C.int for i := 0; i < len(dest); i++ { - var val *C.sqlite3_value - var src any - - rc := C.sqlite3changeset_new(ci.iter, C.int(i), &val) + fn := "" + if old { + fn = "old" + rc = C.sqlite3changeset_old(ci.iter, C.int(i), &val) + } else { + fn = "new" + rc = C.sqlite3changeset_new(ci.iter, C.int(i), &val) + } if rc != C.SQLITE_OK { - return fmt.Errorf("sqlite3changeset_new: %s", C.GoString(C.sqlite3_errstr(rc))) + return fmt.Errorf("sqlite3changeset_%s: %s", fn, C.GoString(C.sqlite3_errstr(rc))) } switch C.sqlite3_value_type(val) { case C.SQLITE_INTEGER: - src = int64(C.sqlite3_value_int64(val)) + dest[i] = int64(C.sqlite3_value_int64(val)) case C.SQLITE_FLOAT: - src = float64(C.sqlite3_value_double(val)) + dest[i] = float64(C.sqlite3_value_double(val)) case C.SQLITE_BLOB: len := C.sqlite3_value_bytes(val) blobptr := C.sqlite3_value_blob(val) - src = C.GoBytes(blobptr, len) + dest[i] = C.GoBytes(blobptr, len) case C.SQLITE_TEXT: - len := C.sqlite3_value_bytes(val) cstrptr := unsafe.Pointer(C.sqlite3_value_text(val)) - src = C.GoBytes(cstrptr, len) + dest[i] = C.GoString((*C.char)(cstrptr)) case C.SQLITE_NULL: - src = nil - } - - err := convertAssign(&dest[i], src) - if err != nil { - return err - } - } - return nil -} - -// Finalize deletes a changeset iterator. -func (ci *ChangesetIterator) Finalize() error { - if ci.iter != nil { - rc := C.sqlite3changeset_finalize(ci.iter) - ci.iter = nil - if rc != C.SQLITE_OK { - return fmt.Errorf("sqlite3changeset_finalize: %s", C.GoString(C.sqlite3_errstr(rc))) + dest[i] = nil } } return nil From 0f37cfbf8423876d35399c7851767e9a1af72525 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Mon, 20 Jan 2025 15:25:21 -0500 Subject: [PATCH 35/39] Test one changeset row --- sqlite3_session_test.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/sqlite3_session_test.go b/sqlite3_session_test.go index 0b5cb476..3e45d094 100644 --- a/sqlite3_session_test.go +++ b/sqlite3_session_test.go @@ -100,25 +100,25 @@ func Test_Changeset_OneRow(t *testing.T) { } defer func() { if err := session.DeleteSession(); err != nil { - t.Errorf("Failed to delete session: %v", err) + t.Fatalf("Failed to delete session: %v", err) } }() + // Attach to the table, insert a row, and capture the iunserted row in a changeset. err = session.AttachSession("test_table") if err != nil { t.Fatalf("Failed to attach session to table: %v", err) } - - _, err = conn.ExecContext(ctx, `INSERT INTO test_table (value) VALUES ('test')`) + _, err = conn.ExecContext(ctx, `INSERT INTO test_table (value) VALUES ('fiona')`) if err != nil { t.Fatalf("Failed to insert row: %v", err) } - changeset, err := NewChangeset(session) if err != nil { t.Fatalf("Failed to generate changeset: %v", err) } + // Prepare to iterate over the changeset. iter, err := NewChangesetIterator(changeset) if err != nil { t.Fatalf("Failed to create changeset iterator: %v", err) @@ -129,6 +129,7 @@ func Test_Changeset_OneRow(t *testing.T) { t.Fatalf("changeset does not contain changes: %v", b) } + // Check table, number of columns changed, the the operation type. tblName, nCol, op, indirect, err := iter.Op() if err != nil { t.Fatalf("Failed to get changeset operation: %v", err) @@ -146,21 +147,23 @@ func Test_Changeset_OneRow(t *testing.T) { t.Fatalf("Expected indirect false, got true") } + // Now, get the new data. dest := make([]any, nCol) if err := iter.New(dest); err != nil { t.Fatalf("Failed to get new row: %v", err) } if v, ok := dest[0].(int64); !ok { t.Fatalf("Expected int64, got %T", dest[0]) - } else if v != 1 { - t.Fatalf("Expected 1, got %d", v) + } else if exp, got := v, int64(1); exp != got { + t.Fatalf("Expected %d, got %d", exp, got) } if v, ok := dest[1].(string); !ok { t.Fatalf("Expected string, got %T", dest[1]) - } else if v != "test" { - t.Fatalf("Expected test, got %s", v) + } else if exp, got := v, "fiona"; exp != got { + t.Fatalf("Expected %s, %s", exp, got) } + // We only inserted one row, so there should be no more changes. if b, err := iter.Next(); err != nil { t.Fatalf("Failed to get next changeset: %v", err) } else if b { From abe49d716dc1980fdda513d053c92aca30ad0f50 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Mon, 20 Jan 2025 15:25:50 -0500 Subject: [PATCH 36/39] Fix typo --- sqlite3_session_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite3_session_test.go b/sqlite3_session_test.go index 3e45d094..d66ffdb5 100644 --- a/sqlite3_session_test.go +++ b/sqlite3_session_test.go @@ -104,7 +104,7 @@ func Test_Changeset_OneRow(t *testing.T) { } }() - // Attach to the table, insert a row, and capture the iunserted row in a changeset. + // Attach to the table, insert a row, and capture the inserted row in a changeset. err = session.AttachSession("test_table") if err != nil { t.Fatalf("Failed to attach session to table: %v", err) From 422b6bf0a27fe09bddd5edb066d92a8400a56724 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Mon, 20 Jan 2025 16:58:39 -0500 Subject: [PATCH 37/39] More session testing --- sqlite3_session_test.go | 132 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) diff --git a/sqlite3_session_test.go b/sqlite3_session_test.go index d66ffdb5..e93b2fa2 100644 --- a/sqlite3_session_test.go +++ b/sqlite3_session_test.go @@ -174,3 +174,135 @@ func Test_Changeset_OneRow(t *testing.T) { t.Fatalf("Failed to finalize changeset iterator: %v", err) } } + +func Test_Changeset_Multi(t *testing.T) { + db, err := sql.Open("sqlite3", ":memory:") + if err != nil { + t.Fatal("Failed to open", err) + } + err = db.Ping() + if err != nil { + t.Fatal("Failed to ping", err) + } + defer db.Close() + + ctx := context.Background() + conn, err := db.Conn(ctx) + if err != nil { + t.Fatal("Failed to get connection to database:", err) + } + defer conn.Close() + + // Create tables, and attach a session. + mustExecute(conn, `CREATE TABLE table1 (id INTEGER PRIMARY KEY, name TEXT, gpa FLOAT, alive boolean)`) + mustExecute(conn, `CREATE TABLE table2 (id INTEGER PRIMARY KEY, age INTEGER)`) + var session *Session + if err := conn.Raw(func(raw any) error { + var err error + session, err = raw.(*SQLiteConn).CreateSession("main") + return err + }); err != nil { + t.Fatal("Failed to create session:", err) + } + defer func() { + if err := session.DeleteSession(); err != nil { + t.Fatalf("Failed to delete session: %v", err) + } + }() + err = session.AttachSession("") + if err != nil { + t.Fatalf("Failed to attach session to table: %v", err) + } + + // Make a bunch of changes. + mustExecute(conn, `INSERT INTO table1 (name, gpa, alive) VALUES ('fiona', 3.5, 1)`) + mustExecute(conn, `INSERT INTO table1 (name, gpa, alive) VALUES ('declan', 2.5, 0)`) + mustExecute(conn, `INSERT INTO table2 (age) VALUES (20)`) + mustExecute(conn, `INSERT INTO table2 (age) VALUES (30)`) + + // Prepare to iterate over the changes. + changeset, err := NewChangeset(session) + if err != nil { + t.Fatalf("Failed to generate changeset: %v", err) + } + iter, err := NewChangesetIterator(changeset) + if err != nil { + t.Fatalf("Failed to create changeset iterator: %v", err) + } + + tt := []struct { + table string + op int + data []any + }{ + { + table: "table1", + op: SQLITE_INSERT, + data: []any{int64(1), "fiona", 3.5, int64(1)}, + }, + { + table: "table1", + op: SQLITE_INSERT, + data: []any{int64(2), "declan", 2.5, int64(0)}, + }, + { + table: "table2", + op: SQLITE_INSERT, + data: []any{int64(1), int64(20)}, + }, + { + table: "table2", + op: SQLITE_INSERT, + data: []any{int64(2), int64(30)}, + }, + } + + for _, v := range tt { + if b, err := iter.Next(); err != nil { + t.Fatalf("Failed to get next changeset: %v", err) + } else if !b { + t.Fatalf("changeset does not contain changes: %v", b) + } + + tblName, nCol, op, _, err := iter.Op() + if err != nil { + t.Fatalf("Failed to get changeset operation: %v", err) + } + if exp, got := v.table, tblName; exp != got { + t.Fatalf("Expected table name '%s', got '%s'", exp, got) + } + if exp, got := len(v.data), nCol; exp != got { + t.Fatalf("Expected %d columns, got %d", exp, got) + } + if exp, got := v.op, op; exp != got { + t.Fatalf("Expected operation %d, got %d", exp, got) + } + + dest := make([]any, nCol) + if err := iter.New(dest); err != nil { + t.Fatalf("Failed to get new row: %v", err) + } + for j, d := range v.data { + if exp, got := d, dest[j]; exp != got { + t.Fatalf("Expected %v (%T) for dest[%d], got %v (%T)", exp, exp, j, got, got) + } + } + } + + if b, err := iter.Next(); err != nil { + t.Fatalf("Failed to get next changeset: %v", err) + } else if b { + t.Fatalf("changeset contains changes: %v", b) + } + + if err := iter.Finalize(); err != nil { + t.Fatalf("Failed to finalize changeset iterator: %v", err) + } +} + +func mustExecute(conn *sql.Conn, stmt string) { + _, err := conn.ExecContext(context.Background(), stmt) + if err != nil { + panic(err) + } +} From 126033a4df7c89a4611ae912c07bfb53fb0b543a Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Mon, 20 Jan 2025 18:29:09 -0500 Subject: [PATCH 38/39] More testing of Session --- sqlite3_session_test.go | 53 +++++++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/sqlite3_session_test.go b/sqlite3_session_test.go index e93b2fa2..e2680c01 100644 --- a/sqlite3_session_test.go +++ b/sqlite3_session_test.go @@ -196,6 +196,7 @@ func Test_Changeset_Multi(t *testing.T) { // Create tables, and attach a session. mustExecute(conn, `CREATE TABLE table1 (id INTEGER PRIMARY KEY, name TEXT, gpa FLOAT, alive boolean)`) mustExecute(conn, `CREATE TABLE table2 (id INTEGER PRIMARY KEY, age INTEGER)`) + mustExecute(conn, `CREATE TABLE table3 (id INTEGER PRIMARY KEY, company TEXT)`) var session *Session if err := conn.Raw(func(raw any) error { var err error @@ -219,6 +220,8 @@ func Test_Changeset_Multi(t *testing.T) { mustExecute(conn, `INSERT INTO table1 (name, gpa, alive) VALUES ('declan', 2.5, 0)`) mustExecute(conn, `INSERT INTO table2 (age) VALUES (20)`) mustExecute(conn, `INSERT INTO table2 (age) VALUES (30)`) + mustExecute(conn, `INSERT INTO table3 (company) VALUES ('foo')`) + mustExecute(conn, `UPDATE table3 SET company = 'bar' WHERE id = 1`) // Prepare to iterate over the changes. changeset, err := NewChangeset(session) @@ -233,31 +236,37 @@ func Test_Changeset_Multi(t *testing.T) { tt := []struct { table string op int - data []any + old []any + new []any }{ { table: "table1", op: SQLITE_INSERT, - data: []any{int64(1), "fiona", 3.5, int64(1)}, + new: []any{int64(1), "fiona", 3.5, int64(1)}, }, { table: "table1", op: SQLITE_INSERT, - data: []any{int64(2), "declan", 2.5, int64(0)}, + new: []any{int64(2), "declan", 2.5, int64(0)}, }, { table: "table2", op: SQLITE_INSERT, - data: []any{int64(1), int64(20)}, + new: []any{int64(1), int64(20)}, }, { table: "table2", op: SQLITE_INSERT, - data: []any{int64(2), int64(30)}, + new: []any{int64(2), int64(30)}, + }, + { + table: "table3", + op: SQLITE_INSERT, + new: []any{int64(1), "bar"}, }, } - for _, v := range tt { + for i, v := range tt { if b, err := iter.Next(); err != nil { t.Fatalf("Failed to get next changeset: %v", err) } else if !b { @@ -271,20 +280,34 @@ func Test_Changeset_Multi(t *testing.T) { if exp, got := v.table, tblName; exp != got { t.Fatalf("Expected table name '%s', got '%s'", exp, got) } - if exp, got := len(v.data), nCol; exp != got { - t.Fatalf("Expected %d columns, got %d", exp, got) - } + // if exp, got := len(v.new), nCol; exp != got { + // t.Fatalf("Expected %d columns, got %d", exp, got) + // } if exp, got := v.op, op; exp != got { t.Fatalf("Expected operation %d, got %d", exp, got) } - dest := make([]any, nCol) - if err := iter.New(dest); err != nil { - t.Fatalf("Failed to get new row: %v", err) + if v.old != nil { + dest := make([]any, nCol) + if err := iter.Old(dest); err != nil { + t.Fatalf("Failed to get old row: %v", err) + } + for j, d := range v.old { + if exp, got := d, dest[j]; exp != got { + t.Fatalf("Test %d, expected %v (%T) for dest[%d], got %v (%T)", i, exp, exp, j, got, got) + } + } } - for j, d := range v.data { - if exp, got := d, dest[j]; exp != got { - t.Fatalf("Expected %v (%T) for dest[%d], got %v (%T)", exp, exp, j, got, got) + + if v.new != nil { + dest := make([]any, nCol) + if err := iter.New(dest); err != nil { + t.Fatalf("Failed to get new row: %v", err) + } + for j, d := range v.new { + if exp, got := d, dest[j]; exp != got { + t.Fatalf("Test %d, expected %v (%T) for new dest[%d], got %v (%T)", i, exp, exp, j, got, got) + } } } } From abfec4480e744c99347872d6705fc636c46a40ba Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Fri, 18 Apr 2025 22:44:16 -0400 Subject: [PATCH 39/39] Remove conflict marker --- sqlite3_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/sqlite3_test.go b/sqlite3_test.go index 22934388..575cdf3d 100644 --- a/sqlite3_test.go +++ b/sqlite3_test.go @@ -1865,7 +1865,6 @@ func TestSetFileControlInt(t *testing.T) { }) } -<<<<<<< HEAD func TestDBConfigNoCkptOnClose(t *testing.T) { fname := TempFilename(t) defer os.Remove(fname)