Feedback for serialize and deserialize

[schickling]

I've started adding support for serialize and deserialize in my local fork of wa-sqlite and would be open to upstreaming this functionality. Before doing so, I'd love to hear some feedback on the implementation as dealing with low-level C from JS is not my area of expertise (yet).

sqlite-api.js

  sqlite3.deserialize = (function() {
    const fname = 'sqlite3_deserialize';
    const f = Module.cwrap(fname, ...decl('nnnnnn:n'));
    return function(db, schema, data, szDb, szBuf, flags) {
      verifyDatabase(db);
      const ptr = Module._sqlite3_malloc(szDb);
      Module.HEAPU8.subarray(ptr).set(data);
      const result = f(db, schema, ptr, szDb, szBuf, flags);
      return result;
    };
  })();

  sqlite3.serialize = (function() {
    const fname = 'sqlite3_serialize';
    const f = Module.cwrap(fname, ...decl('nsnn:n'));
    return function(db, schema, flags) {
      verifyDatabase(db);
      const piSize = tmpPtr[0];
      const address = f(db, schema, piSize, flags);
      const size = Module.getValue(piSize, '*');
      const result = Module.HEAPU8.subarray(address, address + size);
      return result;
    };
  })();

index.d.ts

  deserialize(
    db: number,
    zSchema: string,
    pData: Uint8Array,
    szDb: number,
    szBuf: number,
    mFlags: number
  ): number;

  serialize(
    db: number,
    zSchema: string,
    mFlags: number
  ): Uint8Array;

I've also noticed that when trying to serialize a DB that was previously deserialized, it seems to only work with the SQLITE_SERIALIZE_NOCOPY flag passed into serialize since otherwise address will be 0. This seems cumbersome to ask the app developer to keep track of, so one possible approach could be to hide the flags from serialize (as there are only 2 options anyway: 0 and SQLITE_SERIALIZE_NOCOPY) and automatically try both paths in the implementation.

[schickling]

Update: I just found out about sqlite3_backup, so I'll experiment more with it. Maybe it's a better fit than serialize and deserialize for my use cases.

It seems like sqlite3_backup is indeed what I was looking for, and I'm now using it in combination with serialize and deserialize.

Here is my current implementation:

  sqlite3.deserialize = (function() {
    const fname = 'sqlite3_deserialize';
    const f = Module.cwrap(fname, ...decl('nnnnnn:n'));
    return function(db, schema, data, szDb, szBuf, flags) {
      verifyDatabase(db);
      const ptr = Module._sqlite3_malloc(szDb);
      Module.HEAPU8.subarray(ptr).set(data);
      const result = f(db, schema, ptr, szDb, szBuf, flags);
      return result;
    };
  })();

  const SQLITE_SERIALIZE_NOCOPY = 0x0_01

  sqlite3.serialize = (function() {
    const fname = 'sqlite3_serialize';
    const f = Module.cwrap(fname, ...decl('nsnn:n'));
    return function(db, schema) {
      verifyDatabase(db);
      const piSize = tmpPtr[0];
      let address = f(db, schema, piSize, 0);
      if (address === 0) {
        address = f(db, schema, piSize, SQLITE_SERIALIZE_NOCOPY);
        const size = Module.getValue(piSize, '*');
        const result = Module.HEAPU8.subarray(address, address + size);
        // NOTE Given that the memory is owned by SQLite, we must copy it.
        return new Uint8Array(result.slice());
      } else {
        const size = Module.getValue(piSize, '*');
        const result = Module.HEAPU8.subarray(address, address + size);
        // Here we're getting a copy of the memory, so we can return it directly.
        return new Uint8Array(result);
      }
    };
  })();

  sqlite3.backup = (function() {
    const fInit = Module.cwrap('sqlite3_backup_init', ...decl('nsns:n'));
    const fStep = Module.cwrap('sqlite3_backup_step', ...decl('nn:n'));
    const fFinish = Module.cwrap('sqlite3_backup_finish', ...decl('n:n'));
    return function(dest, destName, source, sourceName) {
      verifyDatabase(dest);
      verifyDatabase(source);
      const backup = fInit(dest, destName, source, sourceName);
      if (backup === 0) {
        const errMsg = Module.ccall('sqlite3_errmsg', 'string', ['number'], [dest]);
        throw new SQLiteError(`backup failed: ${errMsg}`, SQLite.SQLITE_ERROR);
      }
      fStep(backup, -1);
      return fFinish(backup);
    };
  })();

[rhashimoto]

I have not used any of these functions from any language so I'm not familiar with them.

I would probably expose all the backup functions individually if I were going to add that capability. It's less documentation burden if you can just point to the SQLite docs. 😀