diff --git a/package.json b/package.json index a6fb23bc..ed067ec4 100644 --- a/package.json +++ b/package.json @@ -20,17 +20,23 @@ "path-parser": "^6.1.0", "process": "^0.11.10", "stream-browserify": "^3.0.0", + "wa-sqlite": "rhashimoto/wa-sqlite#buildless", "warcio": "^1.5.1" }, "devDependencies": { "@titelmedia/node-fetch": "^3.0.1", + "@types/better-sqlite3": "^7.5.0", "ava": "^3.14.0", + "better-sqlite3": "^7.5.1", "eslint": "^7.23.0", "esm": "^3.2.25", "fake-indexeddb": "^3.0.0", "nyc": "^15.0.1", "raw-loader": "^4.0.2", "test-listen": "^1.1.0", + "ts-loader": "^9.2.8", + "ts-node": "^10.7.0", + "typescript": "^4.6.3", "web-streams-node": "^0.4.0", "webpack": "^5.53.0", "webpack-cli": "^4.8.0", diff --git a/src/api.js b/src/api.js index 506773a4..15da0bb4 100644 --- a/src/api.js +++ b/src/api.js @@ -62,6 +62,7 @@ class API { "curated": "c/:coll/curated/:list", "pages": "c/:coll/pages", "textIndex": "c/:coll/textIndex", + "sqliteFtsSearch": "c/:coll/sqliteFtsSearch", "deletePage": ["c/:coll/page/:page", "DELETE"], }; } @@ -188,6 +189,17 @@ class API { return {}; } } + case "sqliteFtsSearch": { + const coll = await this.collections.getColl(params.coll); + if (!coll) { + return {error: "collection_not_found"}; + } + if (coll.store.sqliteFtsSearch) { + return await coll.store.sqliteFtsSearch(Object.fromEntries(params._query)); + } else { + return {}; + } + } case "curated": { const coll = await this.collections.getColl(params.coll); diff --git a/src/sqlite-fts/.gitignore b/src/sqlite-fts/.gitignore new file mode 100644 index 00000000..4c43fe68 --- /dev/null +++ b/src/sqlite-fts/.gitignore @@ -0,0 +1 @@ +*.js \ No newline at end of file diff --git a/src/sqlite-fts/HttpVFS.ts b/src/sqlite-fts/HttpVFS.ts new file mode 100644 index 00000000..f760817d --- /dev/null +++ b/src/sqlite-fts/HttpVFS.ts @@ -0,0 +1,129 @@ +import SQLiteESMFactory from "wa-sqlite/dist/wa-sqlite-async.mjs"; +import * as SQLite from "wa-sqlite"; +import { Base } from "wa-sqlite/src/VFS.js"; +import * as SQLITE from "wa-sqlite/src/sqlite-constants.js"; +import { LazyUint8Array, HttpVfsProgressEvent } from "./LazyUint8Array"; +type SqliteFtsConfig = { + url: string; + startOffset: number; + length: number; + pageSize: number; +}; + +export class HttpVFS extends Base { + name: string; + config: SqliteFtsConfig; + dbFileId: number = 0; + dbFile: LazyUint8Array; + + constructor( + name: string, + config: SqliteFtsConfig, + progressCallback: (p: HttpVfsProgressEvent) => void + ) { + super(); + this.name = name; + this.config = config; + this.dbFile = new LazyUint8Array({ + fileLength: config.length, + requestChunkSize: config.pageSize, + progressCallback, + rangeMapper: (from, to) => ({ + url: config.url, + fromByte: config.startOffset + from, + toByte: config.startOffset + to, + }), + }); + } + xClose(fileId: number): number | Promise { + if (fileId !== this.dbFileId) throw Error("unknown file id " + fileId); + return SQLITE.SQLITE_OK; + } + xRead( + fileId: number, + pData: { size: number; value: Int8Array }, + iOffset: number + ): number | Promise { + return this.handleAsync(async () => { + if (fileId !== this.dbFileId) + throw Error("xRead: invalid file id " + fileId); + const uint8Array = new Uint8Array( + pData.value.buffer, + pData.value.byteOffset, + pData.value.length + ); + await this.dbFile.copyInto(uint8Array, 0, pData.size, iOffset); + + return SQLITE.SQLITE_OK; + }); + } + xWrite( + fileId: number, + pData: { size: number; value: Int8Array }, + iOffset: number + ): number | Promise { + throw new Error("xWrite not implemented."); + } + xTruncate(fileId: number, iSize: number): number | Promise { + throw new Error("xTruncate not implemented."); + } + xSync(fileId: number, flags: number): number | Promise { + throw new Error("xSync not implemented."); + } + xFileSize( + fileId: number, + pSize64: { set(value: number): void } + ): number | Promise { + if (fileId !== this.dbFileId) + throw new Error(`xFileSize: invalid file id ${fileId}`); + pSize64.set(this.config.length); + return SQLITE.SQLITE_OK; + } + xLock(fileId: number, flags: number): number | Promise { + return SQLITE.SQLITE_OK; + } + xUnlock(fileId: number, flags: number): number | Promise { + return SQLITE.SQLITE_OK; + } + xCheckReservedLock( + fileId: number, + pResOut: { set(value: number): void } + ): number | Promise { + throw new Error("xCheckReservedLockMethod not implemented."); + } + xFileControl( + fileId: number, + flags: number, + pOut: { value: Int8Array } + ): number | Promise { + return SQLITE.SQLITE_NOTFOUND; + } + xDeviceCharacteristics(fileId: number): number | Promise { + return SQLITE.SQLITE_OK; + } + xOpen( + name: string | null, + fileId: number, + flags: number, + pOutFlags: { set(value: number): void } + ): number | Promise { + if (name !== "dummy") throw Error("file name must be dummy"); + this.dbFileId = fileId; + pOutFlags.set(flags); + return SQLITE.SQLITE_OK; + } + xDelete(name: string, syncDir: number): number | Promise { + throw new Error("xDelete not implemented."); + } + xAccess( + name: string, + flags: number, + pResOut: { set(value: any): void } + ): number | Promise { + if (["dummy-journal", "dummy-wal"].includes(name)) { + pResOut.set(0); + return SQLITE.SQLITE_OK; + } + throw new Error(`xAccess(${name}, ${flags}) not implemented.`); + } +} diff --git a/src/sqlite-fts/LazyUint8Array.ts b/src/sqlite-fts/LazyUint8Array.ts new file mode 100644 index 00000000..0eeeb0ae --- /dev/null +++ b/src/sqlite-fts/LazyUint8Array.ts @@ -0,0 +1,362 @@ +// adapted from https://github.com/phiresky/tantivy-wasm/blob/0990c5ffcb1ec40eb58f0821dbb20a0f517bb44e/src/fetch_directory.ts + +export type RangeMapper = ( + fromByte: number, + toByte: number +) => { url: string; fromByte: number; toByte: number }; + +export type HttpVfsProgressEvent = { + fetchBytes: number; +}; +export type LazyFileConfig = { + /** function to map a read request to an url with read request */ + rangeMapper: RangeMapper; + /** must be known beforehand if there's multiple server chunks (i.e. rangeMapper returns different urls) */ + fileLength?: number; + /** chunk size for random access requests (should be same as sqlite page size) */ + requestChunkSize: number; + /** number of virtual read heads. default: 3 */ + maxReadHeads?: number; + /** max read speed for sequential access. default: 5 MiB */ + maxReadSpeed?: number; + /** if true, log all read pages into the `readPages` field for debugging */ + logPageReads?: boolean; + /** if false, only cache read ahead chunks. default true, only set to false if you have your own cache. default true */ + cacheRequestedChunk?: boolean; + /** */ + progressCallback?: (e: HttpVfsProgressEvent) => void; +}; +export type PageReadLog = { + pageno: number; + // if page was already loaded + wasCached: boolean; + // how many pages were prefetched + prefetch: number; + reason: string; +}; +type ReadHead = { startChunk: number; speed: number }; + +function rangeArray(start: number, endInclusive: number) { + return Array(endInclusive - start + 1) + .fill(0) + .map((_, i) => start + i); +} +export class LazyUint8Array { + private serverChecked = false; + private readonly chunks: Uint8Array[] = []; // Loaded chunks. Index is the chunk number + totalFetchedBytes = 0; + totalRequests = 0; + readPages: PageReadLog[] = []; + private _length?: number; + + // LRU list of read heds, max length = maxReadHeads. first is most recently used + private readonly readHeads: ReadHead[] = []; + private readonly _chunkSize: number; + private readonly rangeMapper: RangeMapper; + private readonly maxSpeed: number; + private readonly maxReadHeads: number; + private readonly logPageReads: boolean; + private readonly cacheRequestedChunk: boolean; + private readonly progressCallback?: (e: HttpVfsProgressEvent) => void; + + constructor(config: LazyFileConfig) { + console.log("new lazy uint8array:", config); + this._chunkSize = config.requestChunkSize; + this.maxSpeed = Math.round( + (config.maxReadSpeed || 5 * 1024 * 1024) / this._chunkSize + ); // max 5MiB at once + this.maxReadHeads = config.maxReadHeads ?? 3; + this.rangeMapper = config.rangeMapper; + this.logPageReads = config.logPageReads ?? false; + this.progressCallback = config.progressCallback; + if (config.fileLength) { + this._length = config.fileLength; + } + this.cacheRequestedChunk = config.cacheRequestedChunk ?? true; + } + /** + * efficiently copy the range [start, start + length) from the http file into the + * output buffer at position [outOffset, outOffest + length) + * reads from cache or synchronously fetches via HTTP if needed + */ + async copyInto( + buffer: Uint8Array, + outOffset: number, + length: number, + start: number + ): Promise { + if (start >= this.length) return 0; + length = Math.min(this.length - start, length); + const end = start + length; + await this.ensureChunksCached( + rangeArray((start / this.chunkSize) | 0, (end / this.chunkSize) | 0) + ); + let i = 0; + + while (i < length) { + // {idx: 24, chunkOffset: 24, chunkNum: 0, wantedSize: 16} + const idx = start + i; + const chunkOffset = idx % this.chunkSize; + const chunkNum = (idx / this.chunkSize) | 0; + const wantedSize = Math.min(this.chunkSize, end - idx); + let inChunk = this.getChunk(chunkNum); + if (chunkOffset !== 0 || wantedSize !== this.chunkSize) { + inChunk = inChunk.subarray(chunkOffset, chunkOffset + wantedSize); + } + buffer.set(inChunk, outOffset + i); + i += inChunk.length; + } + return length; + } + + private lastGet = -1; + /* find the best matching existing read head to get the given chunk or create a new one */ + private moveReadHead(wantedChunkNum: number): ReadHead { + for (const [i, head] of this.readHeads.entries()) { + const fetchStartChunkNum = head.startChunk + head.speed; + const newSpeed = Math.min(this.maxSpeed, head.speed * 2); + const wantedIsInNextFetchOfHead = + wantedChunkNum >= fetchStartChunkNum && + wantedChunkNum < fetchStartChunkNum + newSpeed; + if (wantedIsInNextFetchOfHead) { + head.speed = newSpeed; + head.startChunk = fetchStartChunkNum; + if (i !== 0) { + // move head to front + this.readHeads.splice(i, 1); + this.readHeads.unshift(head); + } + return head; + } + } + const newHead: ReadHead = { + startChunk: wantedChunkNum, + speed: 1, + }; + this.readHeads.unshift(newHead); + while (this.readHeads.length > this.maxReadHeads) this.readHeads.pop(); + return newHead; + } + async ensureChunksCached(_chunkIds: number[]) { + if (this.logPageReads) { + for (const cid of _chunkIds) { + if (typeof this.chunks[cid] !== "undefined") { + this.readPages.push({ + pageno: cid, + wasCached: true, + prefetch: 0, + reason: "[no tracking when cached]", + }); + } + } + } + const chunkIds = new Set( + _chunkIds + .filter((c) => typeof this.chunks[c] === "undefined") + .sort((a, b) => a - b) + ); + await this.fetchChunks([...chunkIds]); + } + + // input: sorted list of chunk ids + private async fetchChunks(wantedChunks: number[]) { + if (wantedChunks.length === 0) return; + const wantedChunkRanges: [number, number][] = []; + const last = wantedChunks.slice(1).reduce<[number, number]>( + ([start, end], current) => { + if (end + 1 === current) { + return [start, current]; + } else { + wantedChunkRanges.push([start, end]); + return [current, current]; + } + }, + [wantedChunks[0], wantedChunks[0]] + ); + wantedChunkRanges.push(last); + + const byteRanges: { + chunks: [number, number]; + bytes: [number, number]; + lastChunkSize: number; + }[] = []; + for (const [wantedStartChunk, wantedEndChunk] of wantedChunkRanges) { + const head = this.moveReadHead(wantedStartChunk); + const newStartChunk = head.startChunk; + const newEndChunk = Math.max( + newStartChunk + head.speed - 1, + wantedEndChunk + ); + const startByte = newStartChunk * this.chunkSize; + const wouldEndByte = (newEndChunk + 1) * this.chunkSize - 1; // including this byte + const endByte = Math.min(wouldEndByte, this.length - 1); // if datalength-1 is selected, this is the last block + const shorter = wouldEndByte - endByte; + //console.log("WOLD", wouldEndByte, endByte, shorter, this.chunkSize - shorter) + //console.log("RANGE", newStartChunk, newEndChunk, startByte, endByte); + + byteRanges.push({ + chunks: [newStartChunk, newEndChunk], + bytes: [startByte, endByte], + lastChunkSize: this.chunkSize - shorter, + }); + } + if (this.logPageReads) { + // TODO: improve log fidelity + const totalChunksFetched = byteRanges.reduce( + (a, b) => a + b.chunks[1] - b.chunks[0] + 1, + 0 + ); + this.readPages.push({ + pageno: wantedChunkRanges[0][0], + wasCached: false, + prefetch: totalChunksFetched - 1, + reason: "idk", + }); + } + const bufs = await this.doFetch(byteRanges.map((x) => x.bytes)); + // console.log(`xhr, got ${bufs.length} chunks`); + for (const [rangeIdx, buf] of bufs.entries()) { + let bufIndex = 0; + const { + chunks: [chunkStart, chunkEnd], + lastChunkSize, + } = byteRanges[rangeIdx]; + for (let curChunk = chunkStart; curChunk <= chunkEnd; curChunk++) { + const curSize = curChunk === chunkEnd ? lastChunkSize : this.chunkSize; + //console.log("CURS", curSize, lastChunkSize, this.chunkSize); + const chunk = buf.subarray(bufIndex, bufIndex + curSize); + bufIndex += curSize; + this.chunks[curChunk] = chunk; + } + if (bufIndex !== buf.byteLength) + throw Error( + `left over response data? ${bufIndex} != ${buf.byteLength}` + ); + } + } + /** get the given chunk from cache, throw if not cached */ + private getChunk(wantedChunkNum: number): Uint8Array { + if (typeof this.chunks[wantedChunkNum] === "undefined") { + throw Error( + `chunk not cached? @${wantedChunkNum} ${this.rangeMapper(0, 1).url}` + ); + } else { + return this.chunks[wantedChunkNum]; + } + } + /** verify the server supports range requests and find out file length */ + private checkServer() { + return true; // we assume server works / supports range requests + size is known for this + var xhr = new XMLHttpRequest(); + const url = this.rangeMapper(0, 0).url; + xhr.open("HEAD", url, false); + xhr.setRequestHeader("Accept-Encoding", "identity"); + xhr.send(null); + if (!((xhr.status >= 200 && xhr.status < 300) || xhr.status === 304)) + throw new Error("Couldn't load " + url + ". Status: " + xhr.status); + var datalength = Number(xhr.getResponseHeader("Content-length")); + + var hasByteServing = xhr.getResponseHeader("Accept-Ranges") === "bytes"; + var usesGzip = xhr.getResponseHeader("Content-Encoding") === "gzip"; + + if (!hasByteServing) { + const msg = + "server either does not support byte serving or does not advertise it (`Accept-Ranges: bytes` header missing), or your database is hosted on CORS and the server doesn't mark the accept-ranges header as exposed."; + console.warn(msg, "seen response headers:", xhr.getAllResponseHeaders()); + // throw Error(msg); + } + + if (usesGzip || !datalength) { + console.error("response headers", xhr.getAllResponseHeaders()); + throw Error("server uses gzip or doesn't have length"); + } + + if (!this._length) this._length = datalength; + this.serverChecked = true; + } + get length() { + if (!this.serverChecked) { + try { + this.checkServer(); + } catch (e) { + console.error("checkServer", e); + throw e; + } + } + return this._length!; + } + + get chunkSize() { + if (!this.serverChecked) { + this.checkServer(); + } + return this._chunkSize!; + } + private async doFetch( + ranges: [absoluteFrom: number, absoluteTo: number][] + ): Promise { + // console.log("doXHR", ranges); + const reqs = new Map(); + + for (const [from, to] of ranges) { + this.totalFetchedBytes += to - from; + if (to > this.length - 1) + throw new Error( + "only " + this.length + " bytes available! programmer error!" + ); + const { fromByte, toByte, url } = this.rangeMapper(from, to); + + let r = reqs.get(url); + if (!r) { + r = []; + reqs.set(url, r); + } + r.push([fromByte, toByte]); + } + this.totalRequests += reqs.size; + if (reqs.size > 1) throw Error("chunk split currently not supported"); + + for (const [url, ranges] of reqs) { + const reqSize = ranges.reduce( + (acc, [from, to]) => acc + to - from + 1, + 0 + ); + this.progressCallback?.({ fetchBytes: reqSize }); + console.log( + `[xhr ${url.split("/").slice(-1)[0]} of ${reqSize / 1024} KiB]` + ); + + // TODO: Use mozResponseArrayBuffer, responseStream, etc. if available. + const headers = new Headers(); + if (this.length !== this.chunkSize) + headers.set( + "Range", + "bytes=" + ranges.map(([from, to]) => `${from}-${to}`).join(", ") + ); + + const resp = await fetch(url, { + headers, + }); + if (!resp.ok) + throw new Error("Couldn't load " + url + ". Status: " + resp.status); + const buf = await resp.arrayBuffer(); + if (ranges.length > 1) { + throw Error("not supported right now"); + /*return parseMultipartBuffer( + buf, + ranges.map(([from, to]) => to - from + 1) + );*/ + } else { + return [new Uint8Array(buf)]; + } + } + throw Error("no request??"); + } + public getCachedChunks() { + const chunks = []; + for (let i in this.chunks) { + chunks.push([+i, this.chunks[i]] as const); + } + return chunks; + } +} diff --git a/src/sqlite-fts/SqliteFtsEngine.ts b/src/sqlite-fts/SqliteFtsEngine.ts new file mode 100644 index 00000000..8434327d --- /dev/null +++ b/src/sqlite-fts/SqliteFtsEngine.ts @@ -0,0 +1,149 @@ +// @ts-expect-error +import SQLiteESMFactory from "./vendor/wa-sqlite-async.mjs"; + +import * as SQLite from "wa-sqlite"; +import * as SQLITE from "wa-sqlite/src/sqlite-constants.js"; +import { HttpVFS } from "./HttpVFS"; +import { HttpVfsProgressEvent } from "./LazyUint8Array.js"; +type SqliteFtsConfig = { + url: string; + startOffset: number; + length: number; + pageSize: number; +}; + +type SearchResponse = + | { + type: "progress"; + progress: HttpVfsProgressEvent; + } + | { + type: "row"; + row: any; + } + | { + type: "error"; + message: string; + }; +export class SqliteFtsEngine { + private readonly config: SqliteFtsConfig; + private sqlite3: SQLiteAPI | null = null; + private db: number | null = null; // pointer to database + private progressStream = new ProgressStream(); + private queryIsRunning = false; + constructor(config: SqliteFtsConfig) { + console.log("new SqliteFtsEngine", config); + this.config = config; + } + async initDb() { + if (!this.sqlite3 || !this.db) { + const module = await SQLiteESMFactory(); + this.sqlite3 = SQLite.Factory(module); + this.sqlite3.vfs_register( + new HttpVFS("httpvfs", this.config, (progress) => + this.progressStream.setProgress(progress) + ) + ); + const db = await this.sqlite3.open_v2("dummy", undefined, "httpvfs"); + this.db = db; + console.log("opened db successfully"); + } + Object.assign(globalThis, { sqlite3: this.sqlite3, db: this.db }); // for debugging + return { sqlite3: this.sqlite3, db: this.db }; + } + + async search(matchString: string, limit: number): Promise { + const results = this + .sql`select *, snippet(pages_fts, -1, '', '', '...', 32) from pages_fts where pages_fts match ${matchString} limit ${limit}`; + + return new ReadableStream({ + async pull(controller) { + console.log("[stream] pulling from stream!"); + let value; + try { + const res = await results.next(); + if (res.done) { + console.log("[stream] done!"); + controller.close(); + return; + } + value = res.value; + } catch (e) { + console.error("error while reading query response", e); + value = { type: "error", message: String(e) }; + } + const text = JSON.stringify(value) + "\n"; + controller.enqueue(new TextEncoder().encode(text)); + }, + }); + } + private async *sql( + strings: TemplateStringsArray, + ...insertions: (string | number)[] + ): AsyncIterator { + if (this.queryIsRunning) throw Error("a query is already running"); + this.queryIsRunning = true; + try { + const assembledExpression = strings.join("?"); + const { sqlite3, db } = await this.initDb(); + const str = sqlite3.str_new(db, assembledExpression); + const prepared = await sqlite3.prepare_v2(db, sqlite3.str_value(str)); + if (!prepared) throw Error("statement could not be prepared"); + console.log(`sql: ${assembledExpression} ${JSON.stringify(insertions)}`); + sqlite3.bind_collection(prepared.stmt, insertions); + const columns = sqlite3.column_names(prepared.stmt); + let sqlitestep = sqlite3.step(prepared.stmt); + while (true) { + const resp = await Promise.race([ + this.progressStream.haveProgressEvent, + sqlitestep, + ]); + yield* this.progressStream + .consume() + .map((progress) => ({ type: "progress" as const, progress })); + if (resp === "progress") { + continue; + } + if (resp === SQLITE.SQLITE_DONE) { + sqlite3.finalize(prepared.stmt); + break; + } + if (resp === SQLITE.SQLITE_ROW) { + const row = sqlite3.row(prepared.stmt); + yield { + type: "row", + row: Object.fromEntries(columns.map((c, i) => [c, row[i]])), + }; + sqlitestep = sqlite3.step(prepared.stmt); + continue; + } + throw Error("unknown sqlite state " + resp); + } + } finally { + this.queryIsRunning = false; + } + } +} + +class ProgressStream { + haveProgressEvent!: Promise<"progress">; + private progressEvents: HttpVfsProgressEvent[] = []; + private sendHaveProgressEvent!: () => void; + constructor() { + this.newNextProgressEvent(); + } + private newNextProgressEvent() { + this.haveProgressEvent = new Promise((r) => { + this.sendHaveProgressEvent = () => r("progress"); + }); + } + consume(): HttpVfsProgressEvent[] { + return this.progressEvents.splice(0, this.progressEvents.length); + } + setProgress(p: HttpVfsProgressEvent) { + console.log("SET PROGRESS", p); + this.progressEvents.push(p); + this.sendHaveProgressEvent(); + this.newNextProgressEvent(); + } +} diff --git a/src/sqlite-fts/create-sqlite-fts.ts b/src/sqlite-fts/create-sqlite-fts.ts new file mode 100644 index 00000000..44ae145a --- /dev/null +++ b/src/sqlite-fts/create-sqlite-fts.ts @@ -0,0 +1,83 @@ +// this is a standalone nodejs script. run via `yarn ts-node --project tsconfig.node.json src/sqlite-fts/create-sqlite-fts.ts .../extraPages.jsonl` +import fs from "fs"; +import Database from "better-sqlite3"; + +const mode: "minimal" | "full" = "minimal"; // if minimal: create a contentless table without columnsize and with detail=none +const pageSize = 4096; +const pagesJson = process.argv[2]; +if (!pagesJson) throw Error(`usage: create-sqlite-fts pagesJson`); + +const content = fs + .readFileSync(pagesJson, { encoding: "utf8" }) + .trim() + .split("\n") + .slice(1) // skip first line + .map((e) => JSON.parse(e)); +const pagesSqlite = pagesJson.replace(".jsonl", "-fts.sqlite3"); +console.log("writing result to", pagesSqlite); +const db = new Database(pagesSqlite); + +// todo: this is somewhat specific to english language, maybe also try trigram +// todo: change pgsz option? + +const ftsPageSize = Math.round((4050 / 4096) * pageSize); // +db.exec(` +pragma page_size = ${ftsPageSize}; -- trade off of number of requests that need to be made vs overhead. +pragma journal_mode = WAL; +pragma synchronous = OFF; +create virtual table pages_fts using fts5( + id UNINDEXED, + url UNINDEXED, + ts UNINDEXED, + title, + text, + tokenize = 'porter unicode61' + ${mode === "minimal" ? ", content='pages', columnsize=0, detail=none" : ""} +); +insert into pages_fts(pages_fts, rank) values ('pgsz', ${ftsPageSize}); +`); + +let insert; +if (mode === "minimal") { + // create separate table with the data to prevent sqlite from storing the title and text + db.exec(` + create table pages(id text, url text, ts text, title text, text text); + `); + + const insContent = db.prepare( + `insert into pages (id, url, ts, title, text) values (:id, :url, :ts, :title, :text)` + ); + const insFts = db.prepare( + `insert into pages_fts (rowid, id, url, ts, title, text) values ((select rowid from pages where id = :id), :id, :url, :ts, :title, :text)` + ); + insert = (data: any) => { + const res = insContent.run({ ...data, ts: null, text: null }); + insFts.run({ ...data, rowid: res.lastInsertRowid }); + }; +} else { + const insQuery = db.prepare( + `insert into pages_fts (id, url, ts, title, text) values (:id, :url, :ts, :title, :text)` + ); + insert = (data: any) => insQuery.run(data); +} + +for (const entry of content) { + const entryWithDefaults = { + url: null, + title: null, + ...entry, + }; + try { + insert(entryWithDefaults); + } catch (e: unknown) { + (e as any).context = entryWithDefaults; + throw e; + } +} +db.exec(` +insert into pages_fts(pages_fts) values ('optimize'); -- for every FTS table you have (if you have any) +pragma journal_mode = DELETE; +vacuum; -- reorganize database and apply changed page size +`); + +// todo: select *, snippet(pages_fts, -1, '[[', ']]', '...', 32) from pages_fts where pages_fts match 'hello'; diff --git a/src/sqlite-fts/types.d.ts b/src/sqlite-fts/types.d.ts new file mode 100644 index 00000000..24509d32 --- /dev/null +++ b/src/sqlite-fts/types.d.ts @@ -0,0 +1,4 @@ +module "**/wa-sqlite*" { + function ModuleFactory(config?: object): Promise; + export = ModuleFactory; +} diff --git a/src/sqlite-fts/vendor/wa-sqlite-async.mjs b/src/sqlite-fts/vendor/wa-sqlite-async.mjs new file mode 100644 index 00000000..24886c27 --- /dev/null +++ b/src/sqlite-fts/vendor/wa-sqlite-async.mjs @@ -0,0 +1,118 @@ + +var Module = (() => { + var _scriptDir = import.meta.url; + + return ( +function(Module) { + Module = Module || {}; + + +var e;e||(e=typeof Module !== 'undefined' ? Module : {});var aa,ba;e.ready=new Promise(function(a,b){aa=a;ba=b});var ca=Object.assign({},e),da="./this.program",ea=(a,b)=>{throw b;},ha="object"==typeof window,ia="function"==typeof importScripts,ja="object"==typeof process&&"object"==typeof process.versions&&"string"==typeof process.versions.node,l="",ka,la,ma,fs,na,oa; +if(ja)l=ia?require("path").dirname(l)+"/":__dirname+"/",oa=()=>{na||(fs=require("fs"),na=require("path"))},ka=function(a,b){oa();a=na.normalize(a);return fs.readFileSync(a,b?void 0:"utf8")},ma=a=>{a=ka(a,!0);a.buffer||(a=new Uint8Array(a));return a},la=(a,b,c)=>{oa();a=na.normalize(a);fs.readFile(a,function(d,f){d?c(d):b(f.buffer)})},1{if(noExitRuntime)throw process.exitCode=a,b;b instanceof pa||r("exiting due to exception: "+b);process.exit(a)},e.inspect=function(){return"[Emscripten Module object]"};else if(ha||ia)ia?l=self.location.href:"undefined"!=typeof document&&document.currentScript&&(l=document.currentScript.src),_scriptDir&&(l=_scriptDir),0!==l.indexOf("blob:")?l=l.substr(0,l.replace(/[?#].*/,"").lastIndexOf("/")+1):l="",ka=a=>{var b=new XMLHttpRequest;b.open("GET",a,!1);b.send(null); +return b.responseText},ia&&(ma=a=>{var b=new XMLHttpRequest;b.open("GET",a,!1);b.responseType="arraybuffer";b.send(null);return new Uint8Array(b.response)}),la=(a,b,c)=>{var d=new XMLHttpRequest;d.open("GET",a,!0);d.responseType="arraybuffer";d.onload=()=>{200==d.status||0==d.status&&d.response?b(d.response):c()};d.onerror=c;d.send(null)};var qa=e.print||console.log.bind(console),r=e.printErr||console.warn.bind(console);Object.assign(e,ca);ca=null;e.thisProgram&&(da=e.thisProgram);e.quit&&(ea=e.quit); +var ra;e.wasmBinary&&(ra=e.wasmBinary);var noExitRuntime=e.noExitRuntime||!0;"object"!=typeof WebAssembly&&u("no native wasm support detected"); +function w(a,b,c="i8"){"*"===c.charAt(c.length-1)&&(c="i32");switch(c){case "i1":x[a>>0]=b;break;case "i8":x[a>>0]=b;break;case "i16":sa[a>>1]=b;break;case "i32":y[a>>2]=b;break;case "i64":z=[b>>>0,(A=b,1<=+Math.abs(A)?0>>0:~~+Math.ceil((A-+(~~A>>>0))/4294967296)>>>0:0)];y[a>>2]=z[0];y[a+4>>2]=z[1];break;case "float":ta[a>>2]=b;break;case "double":ua[a>>3]=b;break;default:u("invalid type for setValue: "+c)}} +function C(a,b="i8"){"*"===b.charAt(b.length-1)&&(b="i32");switch(b){case "i1":return x[a>>0];case "i8":return x[a>>0];case "i16":return sa[a>>1];case "i32":return y[a>>2];case "i64":return y[a>>2];case "float":return ta[a>>2];case "double":return Number(ua[a>>3]);default:u("invalid type for getValue: "+b)}return null}var va,E=!1; +function F(a,b,c,d,f){function g(p){0!==k&&wa(k);return"string"===b?G(p):"boolean"===b?!!p:p}var h={string:function(p){var t=0;if(null!==p&&void 0!==p&&0!==p){var v=(p.length<<2)+1;t=xa(v);ya(p,t,v)}return t},array:function(p){var t=xa(p.length);x.set(p,t);return t}};a=e["_"+a];var n=[],k=0;if(d)for(var m=0;m=d);)++c;if(16f?d+=String.fromCharCode(f):(f-=65536,d+=String.fromCharCode(55296|f>>10,56320|f&1023))}}else d+=String.fromCharCode(f)}return d}function G(a,b){return a?Ca(Da,a,b):""} +function Ea(a,b,c,d){if(!(0=h){var n=a.charCodeAt(++g);h=65536+((h&1023)<<10)|n&1023}if(127>=h){if(c>=d)break;b[c++]=h}else{if(2047>=h){if(c+1>=d)break;b[c++]=192|h>>6}else{if(65535>=h){if(c+2>=d)break;b[c++]=224|h>>12}else{if(c+3>=d)break;b[c++]=240|h>>18;b[c++]=128|h>>12&63}b[c++]=128|h>>6&63}b[c++]=128|h&63}}b[c]=0;return c-f}function ya(a,b,c){return Ea(a,Da,b,c)} +function J(a){for(var b=0,c=0;c=d&&(d=65536+((d&1023)<<10)|a.charCodeAt(++c)&1023);127>=d?++b:b=2047>=d?b+2:65535>=d?b+3:b+4}return b}function Fa(a){var b=J(a)+1,c=Ga(b);c&&Ea(a,x,c,b);return c}var Ha,x,Da,sa,y,ta,ua; +function Ia(){var a=va.buffer;Ha=a;e.HEAP8=x=new Int8Array(a);e.HEAP16=sa=new Int16Array(a);e.HEAP32=y=new Int32Array(a);e.HEAPU8=Da=new Uint8Array(a);e.HEAPU16=new Uint16Array(a);e.HEAPU32=new Uint32Array(a);e.HEAPF32=ta=new Float32Array(a);e.HEAPF64=ua=new Float64Array(a)}var Ja=[],Ka=[],La=[],Ma=[];function Na(){var a=e.preRun.shift();Ja.unshift(a)}var K=0,Oa=null,Pa=null;e.preloadedImages={};e.preloadedAudios={}; +function u(a){if(e.onAbort)e.onAbort(a);a="Aborted("+a+")";r(a);E=!0;a=new WebAssembly.RuntimeError(a+". Build with -s ASSERTIONS=1 for more info.");ba(a);throw a;}function Qa(){return L.startsWith("data:application/octet-stream;base64,")}var L;if(e.locateFile){if(L="wa-sqlite-async.wasm",!Qa()){var Ra=L;L=e.locateFile?e.locateFile(Ra,l):l+Ra}}else L=(new URL("wa-sqlite-async.wasm",import.meta.url)).toString(); +function Sa(){var a=L;try{if(a==L&&ra)return new Uint8Array(ra);if(ma)return ma(a);throw"both async and sync fetching of the wasm failed";}catch(b){u(b)}} +function Ta(){if(!ra&&(ha||ia)){if("function"==typeof fetch&&!L.startsWith("file://"))return fetch(L,{credentials:"same-origin"}).then(function(a){if(!a.ok)throw"failed to load wasm binary file at '"+L+"'";return a.arrayBuffer()}).catch(function(){return Sa()});if(la)return new Promise(function(a,b){la(L,function(c){a(new Uint8Array(c))},b)})}return Promise.resolve().then(function(){return Sa()})}var A,z; +function Ua(a){for(;0=b||(b=Math.max(b,c*(1048576>c?2:1.125)>>>0),0!=c&&(b=Math.max(b,256)),c=a.Db,a.Db=new Uint8Array(b),0=a.node.Fb)return 0;a=Math.min(a.node.Fb-f,d);if(8b)throw new N(28);return b},$b:function(a,b,c){P.bc(a.node,b+c);a.node.Fb=Math.max(a.node.Fb,b+c)},Tb:function(a,b,c,d,f,g){if(0!==b)throw new N(28);if(32768!==(a.node.mode&61440))throw new N(43);a=a.node.Db;if(g& +2||a.buffer!==Ha){if(0{a=$a("/",a);if(!a)return{path:"",node:null};b=Object.assign({cc:!0, +Zb:0},b);if(8!!h),!1);for(var c=kb,d="/",f=0;f{for(var b;;){if(a===a.parent)return a=a.Ib.fc,b?"/"!==a[a.length-1]?a+"/"+b:a+b:a;b=b?a.name+"/"+b:a.name;a=a.parent}},qb=(a,b)=>{for(var c= +0,d=0;d>>0)%S.length},rb=a=>{var b=qb(a.parent.id,a.name);if(S[b]===a)S[b]=a.Pb;else for(b=S[b];b;){if(b.Pb===a){b.Pb=a.Pb;break}b=b.Pb}},Q=(a,b)=>{var c;if(c=(c=sb(a,"x"))?c:a.tb.lookup?0:2)throw new N(c,a);for(c=S[qb(a.id,b)];c;c=c.Pb){var d=c.name;if(c.parent.id===a.id&&d===b)return c}return a.tb.lookup(a,b)},hb=(a,b,c,d)=>{a=new tb(a,b,c,d);b=qb(a.parent.id,a.name);a.Pb=S[b];return S[b]=a},ub={r:0,"r+":2,w:577,"w+":578,a:1089,"a+":1090},vb= +a=>{var b=["r","w","rw"][a&3];a&512&&(b+="w");return b},sb=(a,b)=>{if(nb)return 0;if(!b.includes("r")||a.mode&292){if(b.includes("w")&&!(a.mode&146)||b.includes("x")&&!(a.mode&73))return 2}else return 2;return 0},wb=(a,b)=>{try{return Q(a,b),20}catch(c){}return sb(a,"wx")},xb=(a,b,c)=>{try{var d=Q(a,b)}catch(f){return f.Eb}if(a=sb(a,"wx"))return a;if(c){if(16384!==(d.mode&61440))return 54;if(d===d.parent||"/"===pb(d))return 10}else if(16384===(d.mode&61440))return 31;return 0},yb=(a=0,b=4096)=>{for(;a<= +b;a++)if(!R[a])return a;throw new N(33);},Ab=(a,b)=>{zb||(zb=function(){},zb.prototype={});a=Object.assign(new zb,a);b=yb(b,void 0);a.fd=b;return R[b]=a},gb={open:a=>{a.Cb=lb[a.node.rdev].Cb;a.Cb.open&&a.Cb.open(a)},Nb:()=>{throw new N(70);}},cb=(a,b)=>{lb[a]={Cb:b}},Bb=(a,b)=>{var c="/"===b,d=!b;if(c&&kb)throw new N(10);if(!c&&!d){var f=T(b,{cc:!1});b=f.path;f=f.node;if(f.Ob)throw new N(10);if(16384!==(f.mode&61440))throw new N(54);}b={type:a,sc:{},fc:b,lc:[]};a=a.Ib(b);a.Ib=b;b.root=a;c?kb=a:f&& +(f.Ob=b,f.Ib&&f.Ib.lc.push(b))},Cb=(a,b,c)=>{var d=T(a,{parent:!0}).node;a=Ya(a);if(!a||"."===a||".."===a)throw new N(28);var f=wb(d,a);if(f)throw new N(f);if(!d.tb.Sb)throw new N(63);return d.tb.Sb(d,a,b,c)},W=(a,b)=>Cb(a,(void 0!==b?b:511)&1023|16384,0),Db=(a,b,c)=>{"undefined"==typeof c&&(c=b,b=438);Cb(a,b|8192,c)},Eb=(a,b)=>{if(!$a(a))throw new N(44);var c=T(b,{parent:!0}).node;if(!c)throw new N(44);b=Ya(b);var d=wb(c,b);if(d)throw new N(d);if(!c.tb.symlink)throw new N(63);c.tb.symlink(c,b,a)}, +Fb=a=>{var b=T(a,{parent:!0}).node;a=Ya(a);var c=Q(b,a),d=xb(b,a,!0);if(d)throw new N(d);if(!b.tb.rmdir)throw new N(63);if(c.Ob)throw new N(10);b.tb.rmdir(b,a);rb(c)},ob=a=>{a=T(a).node;if(!a)throw new N(44);if(!a.tb.readlink)throw new N(28);return $a(pb(a.parent),a.tb.readlink(a))},Gb=(a,b)=>{a=T(a,{Mb:!b}).node;if(!a)throw new N(44);if(!a.tb.Hb)throw new N(63);return a.tb.Hb(a)},Hb=a=>Gb(a,!0),Ib=(a,b)=>{a="string"==typeof a?T(a,{Mb:!0}).node:a;if(!a.tb.Gb)throw new N(63);a.tb.Gb(a,{mode:b&4095| +a.mode&-4096,timestamp:Date.now()})},Jb=(a,b)=>{if(0>b)throw new N(28);a="string"==typeof a?T(a,{Mb:!0}).node:a;if(!a.tb.Gb)throw new N(63);if(16384===(a.mode&61440))throw new N(31);if(32768!==(a.mode&61440))throw new N(28);var c=sb(a,"w");if(c)throw new N(c);a.tb.Gb(a,{size:b,timestamp:Date.now()})},Lb=(a,b,c,d)=>{if(""===a)throw new N(44);if("string"==typeof b){var f=ub[b];if("undefined"==typeof f)throw Error("Unknown file open mode: "+b);b=f}c=b&64?("undefined"==typeof c?438:c)&4095|32768:0;if("object"== +typeof a)var g=a;else{a=M(a);try{g=T(a,{Mb:!(b&131072)}).node}catch(h){}}f=!1;if(b&64)if(g){if(b&128)throw new N(20);}else g=Cb(a,c,0),f=!0;if(!g)throw new N(44);8192===(g.mode&61440)&&(b&=-513);if(b&65536&&16384!==(g.mode&61440))throw new N(54);if(!f&&(c=g?40960===(g.mode&61440)?32:16384===(g.mode&61440)&&("r"!==vb(b)||b&512)?31:sb(g,vb(b)):44))throw new N(c);b&512&&Jb(g,0);b&=-131713;d=Ab({node:g,path:pb(g),flags:b,seekable:!0,position:0,Cb:g.Cb,pc:[],error:!1},d);d.Cb.open&&d.Cb.open(d);!e.logReadFiles|| +b&1||(Kb||(Kb={}),a in Kb||(Kb[a]=1));return d},Mb=(a,b,c)=>{if(null===a.fd)throw new N(8);if(!a.seekable||!a.Cb.Nb)throw new N(70);if(0!=c&&1!=c&&2!=c)throw new N(28);a.position=a.Cb.Nb(a,b,c);a.pc=[]},Nb=()=>{N||(N=function(a,b){this.node=b;this.nc=function(c){this.Eb=c};this.nc(a);this.message="FS error"},N.prototype=Error(),N.prototype.constructor=N,[44].forEach(a=>{ib[a]=new N(a);ib[a].stack=""}))},Ob,Pb=(a,b)=>{var c=0;a&&(c|=365);b&&(c|=146);return c},Rb=(a,b,c)=>{a= +M("/dev/"+a);var d=Pb(!!b,!!c);Qb||(Qb=64);var f=Qb++<<8|0;cb(f,{open:g=>{g.seekable=!1},close:()=>{c&&c.buffer&&c.buffer.length&&c(10)},read:(g,h,n,k)=>{for(var m=0,q=0;q{for(var m=0;m>2]=d.dev;y[c+4>>2]=0;y[c+8>>2]=d.ino;y[c+12>>2]=d.mode;y[c+16>>2]=d.nlink;y[c+20>>2]=d.uid;y[c+24>>2]=d.gid;y[c+28>>2]=d.rdev;y[c+32>>2]=0;z=[d.size>>>0,(A=d.size,1<=+Math.abs(A)?0>>0:~~+Math.ceil((A-+(~~A>>>0))/4294967296)>>>0:0)];y[c+40>>2]=z[0];y[c+44>>2]=z[1];y[c+48>>2]=4096;y[c+52>>2]=d.blocks;y[c+56>>2]=d.atime.getTime()/1E3|0;y[c+60>>2]= +0;y[c+64>>2]=d.mtime.getTime()/1E3|0;y[c+68>>2]=0;y[c+72>>2]=d.ctime.getTime()/1E3|0;y[c+76>>2]=0;z=[d.ino>>>0,(A=d.ino,1<=+Math.abs(A)?0>>0:~~+Math.ceil((A-+(~~A>>>0))/4294967296)>>>0:0)];y[c+80>>2]=z[0];y[c+84>>2]=z[1];return 0}var Ub=void 0;function Vb(){Ub+=4;return y[Ub-4>>2]}function Y(a){a=R[a];if(!a)throw new N(8);return a} +function Wb(a,b,c){function d(k){return(k=k.toTimeString().match(/\(([A-Za-z ]+)\)$/))?k[1]:"GMT"}var f=(new Date).getFullYear(),g=new Date(f,0,1),h=new Date(f,6,1);f=g.getTimezoneOffset();var n=h.getTimezoneOffset();y[a>>2]=60*Math.max(f,n);y[b>>2]=Number(f!=n);a=d(g);b=d(h);a=Fa(a);b=Fa(b);n>2]=a,y[c+4>>2]=b):(y[c>>2]=b,y[c+4>>2]=a)}function Xb(a,b,c){Xb.jc||(Xb.jc=!0,Wb(a,b,c))}var Yb;Yb=ja?()=>{var a=process.hrtime();return 1E3*a[0]+a[1]/1E6}:()=>performance.now();var Zb={}; +function $b(){if(!ac){var a={USER:"web_user",LOGNAME:"web_user",PATH:"/",PWD:"/",HOME:"/home/web_user",LANG:("object"==typeof navigator&&navigator.languages&&navigator.languages[0]||"C").replace("-","_")+".UTF-8",_:da||"./this.program"},b;for(b in Zb)void 0===Zb[b]?delete a[b]:a[b]=Zb[b];var c=[];for(b in a)c.push(b+"="+a[b]);ac=c}return ac}var ac;function bc(){}function cc(){}function dc(){}function ec(){}function fc(){}function gc(){}function hc(){}function ic(){}function jc(){}function kc(){} +function lc(){}function mc(){}function nc(){}function oc(){}function pc(){}function qc(){}function rc(){}function sc(){}function tc(){}function uc(){}function vc(){}function wc(){}function xc(){}function yc(){}function zc(){}function Ac(){}function Bc(){}function Cc(){}function Dc(){}function Ec(){}function Fc(){}function Gc(){}function Hc(){}function Ic(){}function Jc(){}function Kc(){}function Lc(){}function Mc(a){try{a()}catch(b){u(b)}}var Z=0,H=null,Nc=0,Oc=[],Pc={},Qc={},Rc=0,Sc=null,Tc=[]; +function Uc(a){var b={},c;for(c in a)(function(d){var f=a[d];b[d]="function"==typeof f?function(){Oc.push(d);try{return f.apply(null,arguments)}finally{E||(Oc.pop()===d||u(void 0),H&&1===Z&&0===Oc.length&&(Z=0,Mc(e._asyncify_stop_unwind),"undefined"!=typeof Fibers&&Fibers.tc()))}}:f})(c);return b}function Aa(){return new Promise((a,b)=>{Sc={resolve:a,reject:b}})} +function Vc(){var a=Ga(12300),b=a+12;y[a>>2]=b;y[a+4>>2]=b+12288;b=Oc[0];var c=Pc[b];void 0===c&&(c=Rc++,Pc[b]=c,Qc[c]=b);y[a+8>>2]=c;return a} +function Wc(a){if(!E){if(0===Z){var b=!1,c=!1;a(d=>{if(!E&&(Nc=d||0,b=!0,c)){Z=2;Mc(()=>e._asyncify_start_rewind(H));"undefined"!=typeof Xc&&Xc.Xb.dc&&Xc.Xb.resume();d=!1;try{var f=(0,e.asm[Qc[y[H+8>>2]]])()}catch(n){f=n,d=!0}var g=!1;if(!H){var h=Sc;h&&(Sc=null,(d?h.reject:h.resolve)(f),g=!0)}if(d&&!g)throw f;}});c=!0;b||(Z=1,H=Vc(),Mc(()=>e._asyncify_start_unwind(H)),"undefined"!=typeof Xc&&Xc.Xb.dc&&Xc.Xb.pause())}else 2===Z?(Z=0,Mc(e._asyncify_stop_rewind),Yc(H),H=null,Tc.forEach(d=>{if(!E)try{d()}catch(f){Va(f)}})): +u("invalid state: "+Z);return Nc}}function Zc(a){return Wc(b=>{a().then(b)})}var $c={};function tb(a,b,c,d){a||(a=this);this.parent=a;this.Ib=a.Ib;this.Ob=null;this.id=mb++;this.name=b;this.mode=c;this.tb={};this.Cb={};this.rdev=d}Object.defineProperties(tb.prototype,{read:{get:function(){return 365===(this.mode&365)},set:function(a){a?this.mode|=365:this.mode&=-366}},write:{get:function(){return 146===(this.mode&146)},set:function(a){a?this.mode|=146:this.mode&=-147}}});Nb();S=Array(4096);Bb(P,"/"); +W("/tmp");W("/home");W("/home/web_user");(()=>{W("/dev");cb(259,{read:()=>0,write:(b,c,d,f)=>f});Db("/dev/null",259);bb(1280,eb);bb(1536,fb);Db("/dev/tty",1280);Db("/dev/tty1",1536);var a=Za();Rb("random",a);Rb("urandom",a);W("/dev/shm");W("/dev/shm/tmp")})();(()=>{W("/proc");var a=W("/proc/self");W("/proc/self/fd");Bb({Ib:()=>{var b=hb(a,"fd",16895,73);b.tb={lookup:(c,d)=>{var f=R[+d];if(!f)throw new N(8);c={parent:null,Ib:{fc:"fake"},tb:{readlink:()=>f.path}};return c.parent=c}};return b}},"/proc/self/fd")})(); +var Xc; +(function(){const a=new Map,b=new Map;e.createFunction=function(c,d,f,g,h,n){const k=a.size;a.set(k,{f:n,Lb:h});return F("create_function","number","number string number number number number".split(" "),[c,d,f,g,k,0])};e.createAggregate=function(c,d,f,g,h,n,k){const m=a.size;a.set(m,{step:n,kc:k,Lb:h});return F("create_function","number","number string number number number number".split(" "),[c,d,f,g,m,1])};e.getFunctionUserData=function(c){return b.get(c)};cc=function(c,d,f,g){c=a.get(c);b.set(d, +c.Lb);c.f(d,new Uint32Array(x.buffer,g,f));b.delete(d)};dc=function(c,d,f,g){c=a.get(c);b.set(d,c.Lb);c.step(d,new Uint32Array(x.buffer,g,f));b.delete(d)};bc=function(c,d){c=a.get(c);b.set(d,c.Lb);c.kc(d);b.delete(d)}})(); +(function(){const a="object"===typeof $c,b=new Map,c=new Map,d=new Map,f=a?new Set:null,g=a?new Set:null;class h{constructor(k,m){this.Kb=k;this.type=m}set(k){switch(this.type){case "s":const m=J(k),q=F("sqlite3_malloc","number",["number"],[m+1]);ya(k,q,m+1);w(this.Kb,q,"i32");break;default:w(this.Kb,k,this.type)}}}const n=new Map;uc=function(k,m,q,p){n.set(G(k),{size:m,Rb:Array.from(new Uint32Array(x.buffer,p,q))})};e.createModule=function(k,m,q,p){a&&(q.handleAsync=Zc);const t=b.size;b.set(t,{module:q, +Lb:p});p=0;q.xCreate&&(p|=1);q.xConnect&&(p|=2);q.xBestIndex&&(p|=4);q.xDisconnect&&(p|=8);q.xDestroy&&(p|=16);q.xOpen&&(p|=32);q.xClose&&(p|=64);q.xFilter&&(p|=128);q.xNext&&(p|=256);q.xEof&&(p|=512);q.xColumn&&(p|=1024);q.xRowid&&(p|=2048);q.xUpdate&&(p|=4096);q.xBegin&&(p|=8192);q.xSync&&(p|=16384);q.xCommit&&(p|=32768);q.xRollback&&(p|=65536);q.xFindFunction&&(p|=131072);q.xRename&&(p|=262144);return F("create_module","number",["number","string","number","number"],[k,m,t,p])};kc=function(k,m, +q,p,t,v){m=b.get(m);c.set(t,m);if(a){f.delete(t);for(const B of f)c.delete(B)}p=Array.from(new Uint32Array(x.buffer,p,q)).map(B=>G(B));return m.module.xCreate(k,m.Lb,p,t,new h(v,"s"))};jc=function(k,m,q,p,t,v){m=b.get(m);c.set(t,m);if(a){f.delete(t);for(const B of f)c.delete(B)}p=Array.from(new Uint32Array(x.buffer,p,q)).map(B=>G(B));return m.module.xConnect(k,m.Lb,p,t,new h(v,"s"))};fc=function(k,m){var q=c.get(k),p=n.get("sqlite3_index_info").Rb;const t={};t.nConstraint=C(m+p[0],"i32");t.aConstraint= +[];var v=C(m+p[1],"i32"),B=n.get("sqlite3_index_constraint").size;for(var D=0;Df?-28:Lb(d.path,d.flags,0,f).fd;case 1:case 2:return 0; +case 3:return d.flags;case 4:return f=Vb(),d.flags|=f,0;case 5:return f=Vb(),sa[f+0>>1]=2,0;case 6:case 7:return 0;case 16:case 8:return-28;case 9:return y[ad()>>2]=28,-1;default:return-28}}catch(g){if("undefined"==typeof X||!(g instanceof N))throw g;return-g.Eb}},G:function(a,b){try{var c=Y(a);return Tb(Gb,c.path,b)}catch(d){if("undefined"==typeof X||!(d instanceof N))throw d;return-d.Eb}},A:function(a,b){try{var c=R[a];if(!c)throw new N(8);if(0===(c.flags&2097155))throw new N(28);Jb(c.node,b);return 0}catch(d){if("undefined"== +typeof X||!(d instanceof N))throw d;return-d.Eb}},z:function(a,b){try{if(0===b)return-28;if(b=d)var f=-28;else{var g=ob(b),h=Math.min(d,J(g)),n=x[c+h];ya(g,c,d+1);x[c+h]=n;f=h}return f}catch(k){if("undefined"==typeof X||!(k instanceof +N))throw k;return-k.Eb}},p:function(a){try{return a=G(a),Fb(a),0}catch(b){if("undefined"==typeof X||!(b instanceof N))throw b;return-b.Eb}},F:function(a,b){try{return a=G(a),Tb(Gb,a,b)}catch(c){if("undefined"==typeof X||!(c instanceof N))throw c;return-c.Eb}},n:function(a,b,c){try{b=G(b);b=Sb(a,b);if(0===c){a=b;var d=T(a,{parent:!0}).node;if(!d)throw new N(44);var f=Ya(a),g=Q(d,f),h=xb(d,f,!1);if(h)throw new N(h);if(!d.tb.unlink)throw new N(63);if(g.Ob)throw new N(10);d.tb.unlink(d,f);rb(g)}else 512=== +c?Fb(b):u("Invalid flags passed to unlinkat");return 0}catch(n){if("undefined"==typeof X||!(n instanceof N))throw n;return-n.Eb}},m:function(a,b,c){try{b=G(b);b=Sb(a,b,!0);if(c){var d=y[c>>2],f=y[c+4>>2];g=1E3*d+f/1E6;c+=8;d=y[c>>2];f=y[c+4>>2];h=1E3*d+f/1E6}else var g=Date.now(),h=g;a=g;var n=T(b,{Mb:!0}).node;n.tb.Gb(n,{timestamp:Math.max(a,h)});return 0}catch(k){if("undefined"==typeof X||!(k instanceof N))throw k;return-k.Eb}},f:function(){return Date.now()},K:function(a,b){a=new Date(1E3*y[a>> +2]);y[b>>2]=a.getSeconds();y[b+4>>2]=a.getMinutes();y[b+8>>2]=a.getHours();y[b+12>>2]=a.getDate();y[b+16>>2]=a.getMonth();y[b+20>>2]=a.getFullYear()-1900;y[b+24>>2]=a.getDay();var c=new Date(a.getFullYear(),0,1);y[b+28>>2]=(a.getTime()-c.getTime())/864E5|0;y[b+36>>2]=-(60*a.getTimezoneOffset());var d=(new Date(a.getFullYear(),6,1)).getTimezoneOffset();c=c.getTimezoneOffset();y[b+32>>2]=(d!=c&&a.getTimezoneOffset()==Math.min(c,d))|0},u:function(a,b,c,d,f,g,h){try{var n=R[f];if(!n)return-8;if(0!==(c& +2)&&0===(d&2)&&2!==(n.flags&2097155))throw new N(2);if(1===(n.flags&2097155))throw new N(2);if(!n.Cb.Tb)throw new N(43);var k=n.Cb.Tb(n,a,b,g,c,d);var m=k.Kb;y[h>>2]=k.hc;return m}catch(q){if("undefined"==typeof X||!(q instanceof N))throw q;return-q.Eb}},v:function(a,b,c,d,f,g){try{var h=R[f];if(h&&c&2){var n=Da.slice(a,a+b);h&&h.Cb.Ub&&h.Cb.Ub(h,n,g,b,d)}}catch(k){if("undefined"==typeof X||!(k instanceof N))throw k;return-k.Eb}},L:Xb,e:Yb,d:function(a){var b=Da.length;a>>>=0;if(2147483648=c;c*=2){var d=b*(1+.2/c);d=Math.min(d,a+100663296);var f=Math;d=Math.max(a,d);f=f.min.call(f,2147483648,d+(65536-d%65536)%65536);a:{try{va.grow(f-Ha.byteLength+65535>>>16);Ia();var g=1;break a}catch(h){}g=void 0}if(g)return!0}return!1},x:function(a,b){var c=0;$b().forEach(function(d,f){var g=b+c;f=y[a+4*f>>2]=g;for(g=0;g>0]=d.charCodeAt(g);x[f>>0]=0;c+=d.length+1});return 0},y:function(a,b){var c=$b();y[a>>2]=c.length;var d=0;c.forEach(function(f){d+=f.length+1}); +y[b>>2]=d;return 0},g:function(a){try{var b=Y(a);if(null===b.fd)throw new N(8);b.Wb&&(b.Wb=null);try{b.Cb.close&&b.Cb.close(b)}catch(c){throw c;}finally{R[b.fd]=null}b.fd=null;return 0}catch(c){if("undefined"==typeof X||!(c instanceof N))throw c;return c.Eb}},l:function(a,b){try{var c=Y(a);x[b>>0]=c.tty?2:16384===(c.mode&61440)?3:40960===(c.mode&61440)?7:4;return 0}catch(d){if("undefined"==typeof X||!(d instanceof N))throw d;return d.Eb}},r:function(a,b,c,d){try{a:{for(var f=Y(a),g=a=0;g>2],n=f,k=y[b+8*g>>2],m=h,q=void 0,p=x;if(0>m||0>q)throw new N(28);if(null===n.fd)throw new N(8);if(1===(n.flags&2097155))throw new N(8);if(16384===(n.node.mode&61440))throw new N(31);if(!n.Cb.read)throw new N(28);var t="undefined"!=typeof q;if(!t)q=n.position;else if(!n.seekable)throw new N(70);var v=n.Cb.read(n,p,k,m,q);t||(n.position+=v);var B=v;if(0>B){var D=-1;break a}a+=B;if(B>2]=D;return 0}catch(I){if("undefined"==typeof X||!(I instanceof N))throw I;return I.Eb}}, +j:function(a,b,c,d,f){try{var g=Y(a);a=4294967296*c+(b>>>0);if(-9007199254740992>=a||9007199254740992<=a)return-61;Mb(g,a,d);z=[g.position>>>0,(A=g.position,1<=+Math.abs(A)?0>>0:~~+Math.ceil((A-+(~~A>>>0))/4294967296)>>>0:0)];y[f>>2]=z[0];y[f+4>>2]=z[1];g.Wb&&0===a&&0===d&&(g.Wb=null);return 0}catch(h){if("undefined"==typeof X||!(h instanceof N))throw h;return h.Eb}},B:function(a){try{var b=Y(a);return Wc(function(c){var d=b.node.Ib;d.type.oc? +d.type.oc(d,!1,function(f){f?c(function(){return 29}):c(0)}):c(0)})}catch(c){if("undefined"==typeof X||!(c instanceof N))throw c;return c.Eb}},o:function(a,b,c,d){try{a:{for(var f=Y(a),g=a=0;g>2],k=y[b+(8*g+4)>>2],m=void 0,q=x;if(0>k||0>m)throw new N(28);if(null===h.fd)throw new N(8);if(0===(h.flags&2097155))throw new N(8);if(16384===(h.node.mode&61440))throw new N(31);if(!h.Cb.write)throw new N(28);h.seekable&&h.flags&1024&&Mb(h,0,2);var p="undefined"!=typeof m;if(!p)m= +h.position;else if(!h.seekable)throw new N(70);var t=h.Cb.write(h,q,n,k,m,void 0);p||(h.position+=t);var v=t;if(0>v){var B=-1;break a}a+=v}B=a}y[d>>2]=B;return 0}catch(D){if("undefined"==typeof X||!(D instanceof N))throw D;return D.Eb}},W:bc,pa:cc,fa:dc,ka:ec,M:fc,k:gc,na:hc,ia:ic,da:jc,ea:kc,t:lc,E:mc,oa:nc,i:oc,h:pc,ca:qc,ga:rc,ha:sc,ma:tc,c:uc,ja:vc,la:wc,aa:xc,V:yc,$:zc,ba:Ac,S:Bc,U:Cc,Z:Dc,Y:Ec,R:Fc,Q:Gc,T:Hc,_:Ic,O:Jc,X:Kc,P:Lc}; +(function(){function a(g){g=g.exports;g=Uc(g);e.asm=g;va=e.asm.qa;Ia();Ka.unshift(e.asm.ra);K--;e.monitorRunDependencies&&e.monitorRunDependencies(K);0==K&&(null!==Oa&&(clearInterval(Oa),Oa=null),Pa&&(g=Pa,Pa=null,g()))}function b(g){a(g.instance)}function c(g){return Ta().then(function(h){return WebAssembly.instantiate(h,d)}).then(function(h){return h}).then(g,function(h){r("failed to asynchronously prepare wasm: "+h);u(h)})}var d={a:bd};K++;e.monitorRunDependencies&&e.monitorRunDependencies(K); +if(e.instantiateWasm)try{var f=e.instantiateWasm(d,a);return f=Uc(f)}catch(g){return r("Module.instantiateWasm callback failed with error: "+g),!1}(function(){return ra||"function"!=typeof WebAssembly.instantiateStreaming||Qa()||L.startsWith("file://")||"function"!=typeof fetch?c(b):fetch(L,{credentials:"same-origin"}).then(function(g){return WebAssembly.instantiateStreaming(g,d).then(b,function(h){r("wasm streaming compile failed: "+h);r("falling back to ArrayBuffer instantiation");return c(b)})})})().catch(ba); +return{}})();e.___wasm_call_ctors=function(){return(e.___wasm_call_ctors=e.asm.ra).apply(null,arguments)};e._sqlite3_vfs_find=function(){return(e._sqlite3_vfs_find=e.asm.sa).apply(null,arguments)};e._sqlite3_malloc=function(){return(e._sqlite3_malloc=e.asm.ta).apply(null,arguments)};e._sqlite3_free=function(){return(e._sqlite3_free=e.asm.ua).apply(null,arguments)};e._sqlite3_prepare_v2=function(){return(e._sqlite3_prepare_v2=e.asm.va).apply(null,arguments)}; +e._sqlite3_step=function(){return(e._sqlite3_step=e.asm.wa).apply(null,arguments)};e._sqlite3_column_int=function(){return(e._sqlite3_column_int=e.asm.xa).apply(null,arguments)};e._sqlite3_finalize=function(){return(e._sqlite3_finalize=e.asm.ya).apply(null,arguments)};e._sqlite3_reset=function(){return(e._sqlite3_reset=e.asm.za).apply(null,arguments)};e._sqlite3_clear_bindings=function(){return(e._sqlite3_clear_bindings=e.asm.Aa).apply(null,arguments)}; +e._sqlite3_value_blob=function(){return(e._sqlite3_value_blob=e.asm.Ba).apply(null,arguments)};e._sqlite3_value_text=function(){return(e._sqlite3_value_text=e.asm.Ca).apply(null,arguments)};e._sqlite3_value_bytes=function(){return(e._sqlite3_value_bytes=e.asm.Da).apply(null,arguments)};e._sqlite3_value_double=function(){return(e._sqlite3_value_double=e.asm.Ea).apply(null,arguments)};e._sqlite3_value_int=function(){return(e._sqlite3_value_int=e.asm.Fa).apply(null,arguments)}; +e._sqlite3_value_type=function(){return(e._sqlite3_value_type=e.asm.Ga).apply(null,arguments)};e._sqlite3_result_blob=function(){return(e._sqlite3_result_blob=e.asm.Ha).apply(null,arguments)};e._sqlite3_result_double=function(){return(e._sqlite3_result_double=e.asm.Ia).apply(null,arguments)};e._sqlite3_result_error=function(){return(e._sqlite3_result_error=e.asm.Ja).apply(null,arguments)};e._sqlite3_result_int=function(){return(e._sqlite3_result_int=e.asm.Ka).apply(null,arguments)}; +e._sqlite3_result_null=function(){return(e._sqlite3_result_null=e.asm.La).apply(null,arguments)};e._sqlite3_result_text=function(){return(e._sqlite3_result_text=e.asm.Ma).apply(null,arguments)};e._sqlite3_column_count=function(){return(e._sqlite3_column_count=e.asm.Na).apply(null,arguments)};e._sqlite3_data_count=function(){return(e._sqlite3_data_count=e.asm.Oa).apply(null,arguments)};e._sqlite3_column_blob=function(){return(e._sqlite3_column_blob=e.asm.Pa).apply(null,arguments)}; +e._sqlite3_column_bytes=function(){return(e._sqlite3_column_bytes=e.asm.Qa).apply(null,arguments)};e._sqlite3_column_double=function(){return(e._sqlite3_column_double=e.asm.Ra).apply(null,arguments)};e._sqlite3_column_text=function(){return(e._sqlite3_column_text=e.asm.Sa).apply(null,arguments)};e._sqlite3_column_type=function(){return(e._sqlite3_column_type=e.asm.Ta).apply(null,arguments)};e._sqlite3_column_name=function(){return(e._sqlite3_column_name=e.asm.Ua).apply(null,arguments)}; +e._sqlite3_bind_blob=function(){return(e._sqlite3_bind_blob=e.asm.Va).apply(null,arguments)};e._sqlite3_bind_double=function(){return(e._sqlite3_bind_double=e.asm.Wa).apply(null,arguments)};e._sqlite3_bind_int=function(){return(e._sqlite3_bind_int=e.asm.Xa).apply(null,arguments)};e._sqlite3_bind_null=function(){return(e._sqlite3_bind_null=e.asm.Ya).apply(null,arguments)};e._sqlite3_bind_text=function(){return(e._sqlite3_bind_text=e.asm.Za).apply(null,arguments)}; +e._sqlite3_bind_parameter_count=function(){return(e._sqlite3_bind_parameter_count=e.asm._a).apply(null,arguments)};e._sqlite3_bind_parameter_name=function(){return(e._sqlite3_bind_parameter_name=e.asm.$a).apply(null,arguments)};e._sqlite3_sql=function(){return(e._sqlite3_sql=e.asm.ab).apply(null,arguments)};e._sqlite3_exec=function(){return(e._sqlite3_exec=e.asm.bb).apply(null,arguments)};e._sqlite3_errmsg=function(){return(e._sqlite3_errmsg=e.asm.cb).apply(null,arguments)}; +e._sqlite3_declare_vtab=function(){return(e._sqlite3_declare_vtab=e.asm.db).apply(null,arguments)};e._sqlite3_libversion=function(){return(e._sqlite3_libversion=e.asm.eb).apply(null,arguments)};e._sqlite3_libversion_number=function(){return(e._sqlite3_libversion_number=e.asm.fb).apply(null,arguments)};e._sqlite3_changes=function(){return(e._sqlite3_changes=e.asm.gb).apply(null,arguments)};e._sqlite3_close=function(){return(e._sqlite3_close=e.asm.hb).apply(null,arguments)}; +e._sqlite3_open_v2=function(){return(e._sqlite3_open_v2=e.asm.ib).apply(null,arguments)};var ad=e.___errno_location=function(){return(ad=e.___errno_location=e.asm.jb).apply(null,arguments)},Ga=e._malloc=function(){return(Ga=e._malloc=e.asm.kb).apply(null,arguments)},Yc=e._free=function(){return(Yc=e._free=e.asm.lb).apply(null,arguments)};e._RegisterExtensionFunctions=function(){return(e._RegisterExtensionFunctions=e.asm.mb).apply(null,arguments)}; +e._create_function=function(){return(e._create_function=e.asm.nb).apply(null,arguments)};e._create_module=function(){return(e._create_module=e.asm.ob).apply(null,arguments)};e._register_vfs=function(){return(e._register_vfs=e.asm.pb).apply(null,arguments)};e._getSqliteFree=function(){return(e._getSqliteFree=e.asm.qb).apply(null,arguments)};e._main=function(){return(e._main=e.asm.rb).apply(null,arguments)}; +var jb=e._emscripten_builtin_memalign=function(){return(jb=e._emscripten_builtin_memalign=e.asm.sb).apply(null,arguments)},za=e.stackSave=function(){return(za=e.stackSave=e.asm.ub).apply(null,arguments)},wa=e.stackRestore=function(){return(wa=e.stackRestore=e.asm.vb).apply(null,arguments)},xa=e.stackAlloc=function(){return(xa=e.stackAlloc=e.asm.wb).apply(null,arguments)},dynCall_vi=e.dynCall_vi=function(){return(dynCall_vi=e.dynCall_vi=e.asm.xb).apply(null,arguments)}; +e._asyncify_start_unwind=function(){return(e._asyncify_start_unwind=e.asm.yb).apply(null,arguments)};e._asyncify_stop_unwind=function(){return(e._asyncify_stop_unwind=e.asm.zb).apply(null,arguments)};e._asyncify_start_rewind=function(){return(e._asyncify_start_rewind=e.asm.Ab).apply(null,arguments)};e._asyncify_stop_rewind=function(){return(e._asyncify_stop_rewind=e.asm.Bb).apply(null,arguments)};e.ccall=F; +e.cwrap=function(a,b,c,d){c=c||[];var f=c.every(function(g){return"number"===g});return"string"!==b&&f&&!d?e["_"+a]:function(){return F(a,b,c,arguments,d)}};e.setValue=w;e.getValue=C;e.UTF8ToString=G;e.stringToUTF8=ya;e.lengthBytesUTF8=J;var cd;function pa(a){this.name="ExitStatus";this.message="Program terminated with exit("+a+")";this.status=a}Pa=function dd(){cd||ed();cd||(Pa=dd)}; +function ed(){function a(){if(!cd&&(cd=!0,e.calledRun=!0,!E)){e.noFSInit||Ob||(Ob=!0,Nb(),e.stdin=e.stdin,e.stdout=e.stdout,e.stderr=e.stderr,e.stdin?Rb("stdin",e.stdin):Eb("/dev/tty","/dev/stdin"),e.stdout?Rb("stdout",null,e.stdout):Eb("/dev/tty","/dev/stdout"),e.stderr?Rb("stderr",null,e.stderr):Eb("/dev/tty1","/dev/stderr"),Lb("/dev/stdin",0),Lb("/dev/stdout",1),Lb("/dev/stderr",1));nb=!1;Ua(Ka);Ua(La);aa(e);if(e.onRuntimeInitialized)e.onRuntimeInitialized();if(fd){var b=e._main;try{var c=b(0, +0);if(!noExitRuntime){if(e.onExit)e.onExit(c);E=!0}ea(c,new pa(c))}catch(d){Va(d)}finally{}}if(e.postRun)for("function"==typeof e.postRun&&(e.postRun=[e.postRun]);e.postRun.length;)b=e.postRun.shift(),Ma.unshift(b);Ua(Ma)}}if(!(0