libsql-{sqlite3,ffi}: Add xReadFrameRaw() to the virtual WAL API

This adds a new xReadFrameRaw() function to the virtual WAL API, which
upper layers can use to fetch the full frame, including the page number,
that is useful for appending frames to a WAL.

Author: penberg

libsql-ffi/bundled/bindings/bindgen.rs

@@ -3289,6 +3289,14 @@ pub struct libsql_wal_methods {
             arg3: *mut ::std::os::raw::c_uchar,
         ) -> ::std::os::raw::c_int,
     >,
+    pub xReadFrameRaw: ::std::option::Option<
+        unsafe extern "C" fn(
+            pWal: *mut wal_impl,
+            arg1: ::std::os::raw::c_uint,
+            arg2: ::std::os::raw::c_int,
+            arg3: *mut ::std::os::raw::c_uchar,
+        ) -> ::std::os::raw::c_int,
+    >,
     pub xDbsize:
         ::std::option::Option<unsafe extern "C" fn(pWal: *mut wal_impl) -> ::std::os::raw::c_uint>,
     pub xBeginWriteTransaction:

libsql-ffi/bundled/bindings/session_bindgen.rs

@@ -3795,6 +3795,14 @@ pub struct libsql_wal_methods {
             arg3: *mut ::std::os::raw::c_uchar,
         ) -> ::std::os::raw::c_int,
     >,
+    pub xReadFrameRaw: ::std::option::Option<
+        unsafe extern "C" fn(
+            pWal: *mut wal_impl,
+            arg1: ::std::os::raw::c_uint,
+            arg2: ::std::os::raw::c_int,
+            arg3: *mut ::std::os::raw::c_uchar,
+        ) -> ::std::os::raw::c_int,
+    >,
     pub xDbsize:
         ::std::option::Option<unsafe extern "C" fn(pWal: *mut wal_impl) -> ::std::os::raw::c_uint>,
     pub xBeginWriteTransaction:

libsql-replication/src/injector/sqlite_injector/injector_wal.rs

@@ -110,6 +110,10 @@ impl Wal for InjectorWal {
         self.inner.read_frame(frame_no, buffer)
     }
 
+    fn read_frame_raw(&mut self, frame_no: NonZeroU32, buffer: &mut [u8]) -> Result<()> {
+        self.inner.read_frame_raw(frame_no, buffer)
+    }
+
     fn db_size(&self) -> u32 {
         self.inner.db_size()
     }

libsql-sqlite3/src/wal.c

@@ -3391,6 +3391,29 @@ static int sqlite3WalReadFrame(
   return sqlite3OsRead(pWal->pWalFd, pOut, (nOut>sz ? sz : nOut), iOffset);
 }
 
+/*
+** Read the contents of frame iRead from the wal file into buffer pOut
+** (which is nOut bytes in size). Return SQLITE_OK if successful, or an
+** error code otherwise.
+*/
+static int sqlite3WalReadFrameRaw(
+  Wal *pWal,                      /* WAL handle */
+  u32 iRead,                      /* Frame to read */
+  int nOut,                       /* Size of buffer pOut in bytes */
+  u8 *pOut                        /* Buffer to write page data to */
+){
+  int sz;
+  i64 iOffset;
+  sz = pWal->hdr.szPage;
+  sz = (sz&0xfe00) + ((sz&0x0001)<<16);
+  testcase( sz<=32768 );
+  testcase( sz>=65536 );
+  iOffset = walFrameOffset(iRead, sz);
+  /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */
+  sz += WAL_FRAME_HDRSIZE;
+  return sqlite3OsRead(pWal->pWalFd, pOut, (nOut>sz ? sz : nOut), iOffset);
+}
+
 /*
 ** Return the size of the database in pages (or zero, if unknown).
 */
@@ -4513,6 +4536,7 @@ static int sqlite3WalOpen(
     out->methods.xEndReadTransaction = (void (*)(wal_impl *))sqlite3WalEndReadTransaction;
     out->methods.xFindFrame = (int (*)(wal_impl *, unsigned int, unsigned int *))sqlite3WalFindFrame;
     out->methods.xReadFrame = (int (*)(wal_impl *, unsigned int, int, unsigned char *))sqlite3WalReadFrame;
+    out->methods.xReadFrameRaw = (int (*)(wal_impl *, unsigned int, int, unsigned char *))sqlite3WalReadFrameRaw;
     out->methods.xDbsize = (unsigned int (*)(wal_impl *))sqlite3WalDbsize;
     out->methods.xBeginWriteTransaction = (int (*)(wal_impl *))sqlite3WalBeginWriteTransaction;
     out->methods.xEndWriteTransaction = (int (*)(wal_impl *))sqlite3WalEndWriteTransaction;

libsql-sqlite3/src/wal.h

@@ -56,6 +56,7 @@ typedef struct libsql_wal_methods {
   /* Read a page from the write-ahead log, if it is present. */
   int (*xFindFrame)(wal_impl* pWal, unsigned int, unsigned int *);
   int (*xReadFrame)(wal_impl* pWal, unsigned int, int, unsigned char *);
+  int (*xReadFrameRaw)(wal_impl* pWal, unsigned int, int, unsigned char *);
 
   /* If the WAL is not empty, return the size of the database. */
   unsigned int (*xDbsize)(wal_impl* pWal);

libsql-sqlite3/test/rust_suite/src/virtual_wal.rs

@@ -98,6 +98,14 @@ mod tests {
             self.0.read_frame(frame_no, buffer)
         }
 
+        fn read_frame_raw(
+            &mut self,
+            page_no: NonZeroU32,
+            buffer: &mut [u8],
+        ) -> libsql_sys::wal::Result<()> {
+            self.0.read_frame_raw(page_no, buffer)
+        }
+
         fn frame_count(&self, locked: i32) -> libsql_sys::wal::Result<u32> {
             self.0.frame_count(locked)
         }

libsql-storage/src/lib.rs

@@ -266,6 +266,10 @@ impl Wal for DurableWal {
         Ok(())
     }
 
+    fn read_frame_raw(&mut self, _page_no: std::num::NonZeroU32, _buffer: &mut [u8]) -> Result<()> {
+        todo!()
+    }
+
     fn frame_count(&self, _locked: i32) -> Result<u32> {
         todo!()
     }

libsql-sys/src/wal/either.rs

@@ -39,6 +39,12 @@ macro_rules! create_either {
                 }
             }
 
+            fn read_frame_raw(&mut self, frame_no: std::num::NonZeroU32, buffer: &mut [u8]) -> super::Result<()> {
+                match self {
+                    $( $name::$t(inner) => inner.read_frame_raw(frame_no, buffer) ),*
+                }
+            }
+
             fn db_size(&self) -> u32 {
                 match self {
                     $( $name::$t(inner) => inner.db_size() ),*

libsql-sys/src/wal/ffi.rs

@@ -23,6 +23,7 @@ pub(crate) fn construct_libsql_wal<W: Wal>(wal: *mut W) -> libsql_wal {
             xEndReadTransaction: Some(end_read_transaction::<W>),
             xFindFrame: Some(find_frame::<W>),
             xReadFrame: Some(read_frame::<W>),
+            xReadFrameRaw: Some(read_frame_raw::<W>),
             xDbsize: Some(db_size::<W>),
             xBeginWriteTransaction: Some(begin_write_transaction::<W>),
             xEndWriteTransaction: Some(end_write_transaction::<W>),
@@ -211,6 +212,23 @@ pub unsafe extern "C" fn read_frame<T: Wal>(
     }
 }
 
+pub unsafe extern "C" fn read_frame_raw<T: Wal>(
+    wal: *mut wal_impl,
+    frame: u32,
+    n_out: c_int,
+    p_out: *mut u8,
+) -> i32 {
+    let this = &mut (*(wal as *mut T));
+    let buffer = std::slice::from_raw_parts_mut(p_out, n_out as usize);
+    match this.read_frame_raw(
+        NonZeroU32::new(frame).expect("invalid frame number"),
+        buffer,
+    ) {
+        Ok(_) => SQLITE_OK,
+        Err(code) => code.extended_code,
+    }
+}
+
 pub unsafe extern "C" fn db_size<T: Wal>(wal: *mut wal_impl) -> u32 {
     let this = &mut (*(wal as *mut T));
     this.db_size()

libsql-sys/src/wal/mod.rs

@@ -186,6 +186,8 @@ pub trait Wal {
     fn find_frame(&mut self, page_no: NonZeroU32) -> Result<Option<NonZeroU32>>;
     /// reads frame `frame_no` into buffer.
     fn read_frame(&mut self, frame_no: NonZeroU32, buffer: &mut [u8]) -> Result<()>;
+    /// reads frame `frame_no` including its frame header into buffer.
+    fn read_frame_raw(&mut self, frame_no: NonZeroU32, buffer: &mut [u8]) -> Result<()>;
 
     fn db_size(&self) -> u32;