Refactor vfs setup (#156)

* Refactor vfs setup

This change refactors the VFS setup to make different VFS implementations reusable by other projects. Specifically the memvfs implementation is untied from sqlite-wasm-rs so that it can be used with any rust sqlite project. For this we refactor the crate structure slightly. Instead of having a single sqlite-wasm-rs crate containing all relevant code we now have:

* wsqlite3-sys which contains the bindings and handles linking the actual sqlite library
* rsqlite-vfs which contains vfs definitions and memvfs implementation. This crate conditionally depends on libsqlite3-sys or sqlite-wasm-rs depending on the compilation target. So it can be used for different vfs implementations for different targets
* sqlite-wasm-rs which is now essentially only a shim over the other crates reexporting most of them in the old locations + providing some platform specific impls. This mostly concerns randomness and timestamp access for wasm

Platform specific behavior like randomness and timestamp access for the vfs implementations must be provided by the user by providing a type that handles callbacks for these cases. Otherwise the implementation can be shared. The reason for this is that the implementation of these platform specific function widly differ between platforms, for wasm you need to use web specific methods, for std targets you can use the standard library, for embedded platforms you would likely need to provide custom implementations.

The wsqlite3-sys and rsqlite-vfs crate are implemented as ![no_std] crates.

* pass `slqite3mc` feature

* Efficient copy

* Add WasmOsCallback random fallback

* Remove unused deps

---------

Co-authored-by: Spxg <unsafe@outlook.es>

Author: weiznich

.github/workflows/test.yml

@@ -64,6 +64,7 @@ jobs:
         cd rusqlite
         printf "[patch.crates-io]\n" >> Cargo.toml
         printf "sqlite-wasm-rs = { path = \"..\" }\n" >> Cargo.toml
+        printf "libsqlite3-sys = { path = \"libsqlite3-sys\"}\n" >> Cargo.toml
         WASM_BINDGEN_TEST_TIMEOUT=60 wasm-pack test --node --features modern-full
 
   test_clippy:
@@ -182,3 +183,15 @@ jobs:
         cd tests
         wasm-pack test --chrome --headless
         wasm-pack test --chrome --headless --features sqlite3mc
+        
+  test_vfs_native:
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Test
+        run: |
+          cd crates/rsqlite-vfs
+          cargo test

Cargo.toml

@@ -1,41 +1,28 @@
 [package]
 name = "sqlite-wasm-rs"
 version = "0.5.1"
-links = "wsqlite3"
-edition = "2021"
 authors = ["Spxg <unsafe@outlook.es>"]
-readme = "README.md"
-license = "MIT"
 repository = "https://github.com/Spxg/sqlite-wasm-rs"
 description = "`wasm32-unknown-unknown` bindings to the libsqlite3 library."
 categories = ["development-tools::ffi", "wasm", "database"]
 keywords = ["sqlite", "sqlite-wasm", "wasm", "webassembly", "javascript"]
-rust-version = "1.82.0"
-include = [
-    'src/**/*.rs',
-    'build.rs',
-    'sqlite3/**/*.c',
-    'sqlite3/**/*.h',
-    'sqlite3mc/**/*.c',
-    'sqlite3mc/**/*.h',
-    'shim/**/*.h',
-    'shim/**/*.c',
-    'Cargo.toml',
-    '/README.md',
-    'LICENSE',
-]
+readme.workspace = true
+edition.workspace = true
+license.workspace = true
+rust-version.workspace = true
+include.workspace = true
 
 [dependencies]
 wasm-bindgen = { version = "0.2.104", default-features = false }
 js-sys = { version = "0.3.81", default-features = false }
-thiserror = { version = "2.0.12", default-features = false }
-hashbrown = { version = "0.16.1", default-features = false, features = ["default-hasher"] }
+wsqlite3-sys = { path = "crates/wsqlite3-sys" }
+rsqlite-vfs = { path = "crates/rsqlite-vfs" }
 
 [features]
 # SQLite3MultipleCiphers
 # <https://github.com/utelle/SQLite3MultipleCiphers>
 # <https://utelle.github.io/SQLite3MultipleCiphers>
-sqlite3mc = []
+sqlite3mc = ["wsqlite3-sys/sqlite3mc"]
 
 [dev-dependencies]
 wasm-bindgen-test = "0.3.54"
@@ -53,12 +40,34 @@ members = [
   "tests",
   "crates/sqlite-wasm-libc",
   "crates/sqlite-wasm-vfs",
+  "crates/rsqlite-vfs", 
+  "crates/wsqlite3-sys",
   "extensions/sqlite-vec",
   "examples/implement-a-vfs",
   "examples/nodejs",
-  "examples/use-prebuild-lib"
+  "examples/use-prebuild-lib",
+]
+
+[workspace.package]
+readme = "README.md"
+edition = "2021"
+license = "MIT"
+rust-version = "1.82.0"
+include = [
+    'src/**/*.rs',
+    'build.rs',
+    'sqlite3/**/*.c',
+    'sqlite3/**/*.h',
+    'sqlite3mc/**/*.c',
+    'sqlite3mc/**/*.h',
+    'shim/**/*.h',
+    'shim/**/*.c',
+    'Cargo.toml',
+    '/README.md',
+    'LICENSE',
 ]
 
+
 [patch.crates-io]
 sqlite-wasm-rs = { path = "./" }
 sqlite-wasm-vfs = { path = "./crates/sqlite-wasm-vfs" }

crates/rsqlite-vfs/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "rsqlite-vfs"
+version = "0.1.0"
+readme.workspace = true
+edition.workspace = true
+license.workspace = true
+rust-version.workspace = true
+include.workspace = true
+
+[dependencies]
+hashbrown = { version = "0.16.1", default-features = false, features = ["default-hasher"] }
+thiserror = { version = "2.0.12", default-features = false }
+
+[target.'cfg(not(all(target_family = "wasm", target_os = "unknown")))'.dependencies]
+# libsqlite3-sys doesn't compile without features
+# see https://github.com/rusqlite/rusqlite/issues/1205
+libsqlite3-sys = { version = "0.35.0", default-features = false, features = ["vcpkg", "pkg-config"] }
+
+[target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies]
+wsqlite3-sys = { default-features = false, path = "../wsqlite3-sys" }
+
+[dev-dependencies]
+rand = { version = "0.9" }

src/utils.rs crates/rsqlite-vfs/src/lib.rssrc/utils.rs renamed to crates/rsqlite-vfs/src/lib.rs

@@ -1,13 +1,33 @@
 //! Low-level utilities, traits, and macros for implementing custom SQLite Virtual File Systems (VFS).
+//!
+//! This crate also contains a simple operating system independent memvfs implementation
+#![no_std]
+#![allow(non_upper_case_globals)]
+#![allow(non_camel_case_types)]
+#![allow(non_snake_case)]
+#![cfg_attr(target_feature = "atomics", feature(stdarch_wasm_atomic_wait))]
 
-use crate::bindings::*;
+extern crate alloc;
+
+#[cfg(not(all(target_family = "wasm", target_os = "unknown")))]
+extern crate libsqlite3_sys as bindings;
+#[cfg(all(target_family = "wasm", target_os = "unknown"))]
+extern crate wsqlite3_sys as bindings;
+
+use self::bindings::*;
+
+// libsqlite3-sys misses this type
+#[cfg(not(all(target_family = "wasm", target_os = "unknown")))]
+#[allow(non_camel_case_types)]
+type sqlite3_filename = *const ::core::ffi::c_char;
 
 use alloc::string::String;
 use alloc::vec::Vec;
 use alloc::{boxed::Box, ffi::CString};
 use alloc::{format, vec};
 use core::{cell::RefCell, ffi::CStr, ops::Deref};
-use js_sys::{Date, Math, Number};
+
+pub mod memvfs;
 
 /// A macro to return a specific SQLite error code if a condition is true.
 ///
@@ -70,9 +90,20 @@ macro_rules! unused {
 pub const SQLITE3_HEADER: &str = "SQLite format 3";
 
 /// Generates a random, temporary filename, typically used when SQLite requests a file with a NULL name.
-pub fn random_name() -> String {
-    let random = Number::from(Math::random()).to_string(36).unwrap();
-    random.slice(2, random.length()).as_string().unwrap()
+pub fn random_name(randomness: fn(&mut [u8])) -> String {
+    const GEN_ASCII_STR_CHARSET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ\
+                abcdefghijklmnopqrstuvwxyz\
+                0123456789";
+    const GEN_LEN: u8 = GEN_ASCII_STR_CHARSET.len() as u8;
+    let mut random_buffer = [0; 32];
+    randomness(&mut random_buffer);
+    random_buffer
+        .into_iter()
+        .map(|e| {
+            let idx = e.saturating_sub(GEN_LEN * (e / GEN_LEN));
+            GEN_ASCII_STR_CHARSET[idx as usize] as char
+        })
+        .collect()
 }
 
 /// An in-memory file implementation that stores data in fixed-size chunks. Suitable for temporary files.
@@ -433,6 +464,9 @@ pub trait SQLiteVfs<IO: SQLiteIoMethods> {
     const VERSION: ::core::ffi::c_int;
     const MAX_PATH_SIZE: ::core::ffi::c_int = 1024;
 
+    fn random(buf: &mut [u8]);
+    fn epoch_timestamp_in_ms() -> i64;
+
     fn vfs(
         vfs_name: *const ::core::ffi::c_char,
         app_data: *mut VfsAppData<IO::AppData>,
@@ -452,11 +486,11 @@ pub trait SQLiteVfs<IO: SQLiteIoMethods> {
             xDlError: None,
             xDlSym: None,
             xDlClose: None,
-            xRandomness: Some(x_methods_shim::xRandomness),
+            xRandomness: Some(Self::xRandomness),
             xSleep: Some(x_methods_shim::xSleep),
-            xCurrentTime: Some(x_methods_shim::xCurrentTime),
+            xCurrentTime: Some(Self::xCurrentTime),
             xGetLastError: Some(Self::xGetLastError),
-            xCurrentTimeInt64: Some(x_methods_shim::xCurrentTimeInt64),
+            xCurrentTimeInt64: Some(Self::xCurrentTimeInt64),
             xSetSystemCall: None,
             xGetSystemCall: None,
             xNextSystemCall: None,
@@ -483,7 +517,7 @@ pub trait SQLiteVfs<IO: SQLiteIoMethods> {
         let app_data = IO::Store::app_data(pVfs);
 
         let name = if zName.is_null() {
-            random_name()
+            random_name(Self::random)
         } else {
             check_result!(CStr::from_ptr(zName).to_str()).into()
         };
@@ -598,6 +632,35 @@ pub trait SQLiteVfs<IO: SQLiteIoMethods> {
         }
         code
     }
+
+    /// <https://github.com/sqlite/sqlite/blob/fb9e8e48fd70b463fb7ba6d99e00f2be54df749e/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js#L951>
+    unsafe extern "C" fn xRandomness(
+        _pVfs: *mut sqlite3_vfs,
+        nByte: ::core::ffi::c_int,
+        zOut: *mut ::core::ffi::c_char,
+    ) -> ::core::ffi::c_int {
+        let slice = core::slice::from_raw_parts_mut(zOut.cast(), nByte as usize);
+        Self::random(slice);
+        nByte
+    }
+
+    /// <https://github.com/sqlite/sqlite/blob/fb9e8e48fd70b463fb7ba6d99e00f2be54df749e/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js#L870>
+    unsafe extern "C" fn xCurrentTime(
+        _pVfs: *mut sqlite3_vfs,
+        pTimeOut: *mut f64,
+    ) -> ::core::ffi::c_int {
+        *pTimeOut = 2440587.5 + (Self::epoch_timestamp_in_ms() as f64 / 86400000.0);
+        SQLITE_OK
+    }
+
+    /// <https://github.com/sqlite/sqlite/blob/fb9e8e48fd70b463fb7ba6d99e00f2be54df749e/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js#L877>
+    unsafe extern "C" fn xCurrentTimeInt64(
+        _pVfs: *mut sqlite3_vfs,
+        pOut: *mut sqlite3_int64,
+    ) -> ::core::ffi::c_int {
+        *pOut = ((2440587.5 * 86400000.0) + Self::epoch_timestamp_in_ms() as f64) as sqlite3_int64;
+        SQLITE_OK
+    }
 }
 
 /// A trait that abstracts the `sqlite3_io_methods` struct, allowing for a more idiomatic Rust implementation.
@@ -839,36 +902,6 @@ pub mod x_methods_shim {
     ) -> ::core::ffi::c_int {
         SQLITE_OK
     }
-
-    /// <https://github.com/sqlite/sqlite/blob/fb9e8e48fd70b463fb7ba6d99e00f2be54df749e/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js#L951>
-    pub unsafe extern "C" fn xRandomness(
-        _pVfs: *mut sqlite3_vfs,
-        nByte: ::core::ffi::c_int,
-        zOut: *mut ::core::ffi::c_char,
-    ) -> ::core::ffi::c_int {
-        for i in 0..nByte as usize {
-            *zOut.add(i) = (Math::random() * 255000.0) as _;
-        }
-        nByte
-    }
-
-    /// <https://github.com/sqlite/sqlite/blob/fb9e8e48fd70b463fb7ba6d99e00f2be54df749e/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js#L870>
-    pub unsafe extern "C" fn xCurrentTime(
-        _pVfs: *mut sqlite3_vfs,
-        pTimeOut: *mut f64,
-    ) -> ::core::ffi::c_int {
-        *pTimeOut = 2440587.5 + (Date::new_0().get_time() / 86400000.0);
-        SQLITE_OK
-    }
-
-    /// <https://github.com/sqlite/sqlite/blob/fb9e8e48fd70b463fb7ba6d99e00f2be54df749e/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js#L877>
-    pub unsafe extern "C" fn xCurrentTimeInt64(
-        _pVfs: *mut sqlite3_vfs,
-        pOut: *mut sqlite3_int64,
-    ) -> ::core::ffi::c_int {
-        *pOut = ((2440587.5 * 86400000.0) + Date::new_0().get_time()) as sqlite3_int64;
-        SQLITE_OK
-    }
 }
 
 #[derive(thiserror::Error, Debug)]
@@ -1050,7 +1083,7 @@ pub mod test_suite {
     }
 }
 
-#[cfg(test)]
+#[cfg(all(test, target_arch = "wasm32"))]
 mod tests {
     use super::{MemChunksFile, VfsFile};
     use wasm_bindgen_test::wasm_bindgen_test;
@@ -1104,3 +1137,16 @@ mod tests {
         assert!(file.chunks.len() == 0);
     }
 }
+
+#[cfg(not(all(test, target_arch = "wasm32")))]
+#[test]
+fn random_name_is_valid() {
+    fn random(buf: &mut [u8]) {
+        rand::fill(buf);
+    }
+    let name_1 = random_name(random);
+    let name_2 = random_name(random);
+    assert!(name_1.is_ascii(), "Expected an ascii-name: `{name_1}`");
+    assert!(name_2.is_ascii(), "Expected an ascii-name: `{name_2}`");
+    assert_ne!(name_1, name_2);
+}