diff --git a/rollup.config.js b/rollup.config.js index ca2560e..4ef40eb 100644 --- a/rollup.config.js +++ b/rollup.config.js @@ -45,5 +45,9 @@ export default [ getConfig('src/indexeddb/backend.js', 'indexeddb-backend.js'), getConfig('src/indexeddb/main-thread.js', 'indexeddb-main-thread.js'), getConfig('src/indexeddb/backend.js', 'indexeddb-backend.js', true), - getConfig('src/indexeddb/main-thread.js', 'indexeddb-main-thread.js', true) + getConfig('src/indexeddb/main-thread.js', 'indexeddb-main-thread.js', true), + getConfig('src/webkitFileSystem/backend.js', 'webkitFileSystem-backend.js'), + getConfig('src/webkitFileSystem/main-thread.js', 'webkitFileSystem-main-thread.js'), + getConfig('src/webkitFileSystem/backend.js', 'webkitFileSystem-backend.js', true), + getConfig('src/webkitFileSystem/main-thread.js', 'webkitFileSystem-main-thread.js', true) ]; diff --git a/src/examples/bench/main.worker.js b/src/examples/bench/main.worker.js index 5b5b955..f9e7b23 100644 --- a/src/examples/bench/main.worker.js +++ b/src/examples/bench/main.worker.js @@ -2,6 +2,7 @@ import initSqlJs from '@jlongster/sql.js'; import { SQLiteFS } from '../..'; import MemoryBackend from '../../memory/backend'; import IndexedDBBackend from '../../indexeddb/backend'; +import webkitFileSystemBackend from '../../webkitFileSystem/backend'; import * as queries from './queries'; import * as rawIDBQueries from './queries-raw-idb'; @@ -9,12 +10,13 @@ import * as rawIDBQueries from './queries-raw-idb'; let currentBackendType = 'idb'; let cacheSize = 0; let pageSize = 4096; -let dbName = `db21.sqlite`; +let dbName = `db23.sqlite`; let recordProfile = false; let useRawIDB = false; let memoryBackend = new MemoryBackend({}); let idbBackend = new IndexedDBBackend(); +let fsBackend = new webkitFileSystemBackend(); let sqlFS; // Helper methods @@ -23,7 +25,7 @@ let SQL = null; async function init() { if (SQL == null) { SQL = await initSqlJs({ locateFile: file => file }); - sqlFS = new SQLiteFS(SQL.FS, idbBackend); + sqlFS = new SQLiteFS(SQL.FS, fsBackend); SQL.register_for_idb(sqlFS); if (typeof SharedArrayBuffer === 'undefined') { @@ -172,7 +174,7 @@ async function populateSmall() { async function populateLarge() { clearTimings(); - let count = 400000; + let count = 50000; if (currentBackendType === 'memory') { output( 'Cannot write 1,000,000 items to memory backend, reducing to 100,000' @@ -329,6 +331,8 @@ if (typeof self !== 'undefined') { // but it works for the demo if (currentBackendType === 'memory') { sqlFS.backend = memoryBackend; + } else if (currentBackendType === 'fs') { + sqlFS.backend = fsBackend; } else { sqlFS.backend = idbBackend; } diff --git a/src/indexeddb/backend.js b/src/indexeddb/backend.js index eaf4d7b..6a91d17 100644 --- a/src/indexeddb/backend.js +++ b/src/indexeddb/backend.js @@ -1,6 +1,5 @@ import { File } from '../sqlite-file'; import * as perf from 'perf-deets'; -import { LOCK_TYPES, getPageSize } from '../sqlite-util'; import { FileOps } from './file-ops'; import { FileOpsFallback } from './file-ops-fallback'; diff --git a/src/indexeddb/file-ops.js b/src/indexeddb/file-ops.js index d8b2548..f149917 100644 --- a/src/indexeddb/file-ops.js +++ b/src/indexeddb/file-ops.js @@ -1,4 +1,4 @@ -import { Reader, Writer } from './shared-channel'; +import { Reader, Writer } from '../shared-channel'; function positionToKey(pos, blockSize) { // We are forced to round because of floating point error. `pos` diff --git a/src/indexeddb/worker.js b/src/indexeddb/worker.js index 480aaa6..676b632 100644 --- a/src/indexeddb/worker.js +++ b/src/indexeddb/worker.js @@ -1,4 +1,4 @@ -import { Reader, Writer } from './shared-channel'; +import { Reader, Writer } from '../shared-channel'; import * as perf from 'perf-deets'; import { LOCK_TYPES, isSafeToWrite } from '../sqlite-util'; diff --git a/src/indexeddb/shared-channel.js b/src/shared-channel.js similarity index 100% rename from src/indexeddb/shared-channel.js rename to src/shared-channel.js diff --git a/src/indexeddb/shared-channel.test.js b/src/shared-channel.test.js similarity index 100% rename from src/indexeddb/shared-channel.test.js rename to src/shared-channel.test.js diff --git a/src/sqlite-file.js b/src/sqlite-file.js index 74337e1..1613346 100644 --- a/src/sqlite-file.js +++ b/src/sqlite-file.js @@ -122,6 +122,7 @@ export class File { open() { this.ops.open(); let meta = this.ops.readMeta(); + console.log('got meta', meta); // It's possible that `setattr` has already been called if opening // the file in a mode that truncates it to 0 @@ -216,7 +217,7 @@ export class File { } write(bufferView, offset, length, position) { - // console.log('writing', this.filename, offset, length, position); + console.log('writing', this.filename, offset, length, position); if (this.meta.blockSize == null) { // We don't have a block size yet (an empty file). The first diff --git a/src/webkitFileSystem/backend.js b/src/webkitFileSystem/backend.js new file mode 100644 index 0000000..a4c3a75 --- /dev/null +++ b/src/webkitFileSystem/backend.js @@ -0,0 +1,100 @@ +import { File } from '../sqlite-file'; +import { getPageSize } from '../sqlite-util'; +import * as perf from 'perf-deets'; + +class FileOps { + constructor(filename, rootEntry) { + this.filename = filename; + this.rootEntry = rootEntry; + } + + getDatabaseName() { + return this.filename.replace(/\//g, '-'); + } + + lock(lockType) { + return true; + } + + unlock(lockType) { + return true; + } + + delete() { + console.log('deleting'); + this.entry.remove(); + } + + open() { + this.entry = this.rootEntry.getFile(this.getDatabaseName(), { + create: true + }); + } + + close() { + // close + } + + readMeta() { + let file = this.entry.file(); + let reader = new FileReaderSync(); + + let blockSize = null; + if (file.size > 0) { + let data = reader.readAsArrayBuffer(file.slice(16, 18)); + let arr = new Uint16Array(data); + blockSize = arr[0] * 256; + } + + return { size: file.size, blockSize: blockSize == 0 ? null : blockSize }; + } + + writeMeta(meta) { + let writer = this.entry.createWriter(); + writer.truncate(meta.size); + return 0; + } + + readBlocks(positions, blockSize) { + let res = []; + let file = this.entry.file(); + let reader = new FileReaderSync(); + for (let pos of positions) { + let data = reader.readAsArrayBuffer(file.slice(pos, pos + blockSize)); + res.push({ + pos, + data: data.byteLength === 0 ? new ArrayBuffer(blockSize) : data + }); + } + return res; + } + + writeBlocks(writes, blockSize) { + for (let write of writes) { + let writer = this.entry.createWriter(); + writer.seek(write.pos); + writer.write(new Blob([write.data], { type: 'octet/stream' })); + } + return 0; + } +} + +export default class webkitFileSystemBackend { + constructor() { + // TODO: need to do `navigator.webkitPersistentStorage.requestQuota(10000000, function (quota) { console.log(quota) })` + this.FS = self.webkitRequestFileSystemSync(self.PERSISTENT, 100000); + this.rootEntry = this.FS.root.getDirectory('dbs', { create: true }); + } + + createFile(filename) { + return new File(filename, new FileOps(filename, this.rootEntry)); + } + + startProfile() { + perf.start(); + } + + stopProfile() { + perf.stop(); + } +}