Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions .eslintrc.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,15 @@ module.exports = {
node: true,
es2021: true,
jest: true,
worker: true
worker: true,
},
extends: 'eslint:recommended',
parserOptions: {
ecmaVersion: 12,
sourceType: 'module'
sourceType: 'module',
},
rules: {
'no-unused-vars': 'off',
'no-inner-declarations': 'off'
}
'no-inner-declarations': 'off',
},
};
23 changes: 23 additions & 0 deletions .github/workflows/node.js.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
# This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions

name: Node.js CI

on: [push, pull_request]

jobs:
build:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v2
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v2
with:
node-version: 15.x
- uses: actions/cache@v2
with:
path: '**/node_modules'
key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }}
- run: yarn
- run: yarn lint
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
node_modules
.DS_Store
dist
dist
.eslintcache
3 changes: 3 additions & 0 deletions .prettierrc.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"singleQuote": true
}
17 changes: 8 additions & 9 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@

This is an absurd project.

It implements a backend for [sql.js](https://github.com/sql-js/sql.js/) (sqlite3 compiled for the web) that treats IndexedDB like a disk and stores data in blocks there. That means your sqlite3 database is persisted. And not in the terrible way of reading and writing the whole image at once -- it reads and writes your db in small chunks.
Expand Down Expand Up @@ -47,7 +46,7 @@ import { SQLiteFS } from 'absurd-sql';
import IndexedDBBackend from 'absurd-sql/dist/indexeddb-backend';

async function run() {
let SQL = await initSqlJs({ locateFile: file => file });
let SQL = await initSqlJs({ locateFile: (file) => file });
let sqlFS = new SQLiteFS(SQL.FS, new IndexedDBBackend());
SQL.register_for_idb(sqlFS);

Expand All @@ -67,16 +66,16 @@ async function run() {
PRAGMA journal_mode=MEMORY;
`);

// Your code
// Your code
}
```

## Requirements

Because this uses `SharedArrayBuffer` and the `Atomics` API, there are some requirement for code to run.

* It must be run in a worker thread (you shouldn't block the main thread with queries anyway)
* Your server must respond with the following headers:
- It must be run in a worker thread (you shouldn't block the main thread with queries anyway)
- Your server must respond with the following headers:

```
Cross-Origin-Opener-Policy: same-origin
Expand Down Expand Up @@ -113,7 +112,7 @@ Read [this blog post](https://jlongster.com/future-sql-web) for more details.

There are several things that could be done:

* Add a bunch more tests
* Implement a `webkitFileSystem` backend
* I already started it [here](https://gist.github.com/jlongster/ec00ddbb47b4b29897ab5939b8e32fbe), but initial results showed that it was way slower?
* Bug fixes
- Add a bunch more tests
- Implement a `webkitFileSystem` backend
- I already started it [here](https://gist.github.com/jlongster/ec00ddbb47b4b29897ab5939b8e32fbe), but initial results showed that it was way slower?
- Bug fixes
6 changes: 3 additions & 3 deletions jest.config.js
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
module.exports = {
transformIgnorePatterns: [
// Change MODULE_NAME_HERE to your module that isn't being compiled
'/node_modules/(?!perf-deets).+\\.js$'
'/node_modules/(?!perf-deets).+\\.js$',
],
moduleNameMapper: {
'perf-deets': 'perf-deets/noop'
}
'perf-deets': 'perf-deets/noop',
},
};
10 changes: 9 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,13 @@
"build": "rm -rf dist && rollup -c rollup.config.js",
"jest": "jest",
"jest-debug": "jest --runInBand --useStderr",
"serve": "cd src/examples && ../../node_modules/.bin/webpack serve"
"serve": "cd src/examples && ../../node_modules/.bin/webpack serve",
"lint": "run-p lint:*",
"lint:eslint": "eslint \"src/**/*.{ts,js}\"",
"lint:prettier": "yarn prettier --check .",

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@quolpr why separate eslint from prettier? Ideally prettier is just a plugin from eslint so they are executed together and there's no need to separate the commands. You can have lint and lint:fix and that's it.

"fix": "run-s fix:eslint fix:prettier",
"fix:eslint": "yarn lint:eslint --fix",
"fix:prettier": "yarn lint:prettier --write"
},
"devDependencies": {
"@babel/plugin-transform-modules-commonjs": "^7.14.5",
Expand All @@ -20,7 +26,9 @@
"fast-check": "^2.17.0",
"html-webpack-plugin": "^5.3.2",
"jest": "^27.0.5",
"npm-run-all": "^4.1.5",
"perf-deets": "^1.0.17",
"prettier": "^2.4.1",
"rollup": "^2.53.1",
"rollup-plugin-extensions": "^0.1.0",
"rollup-plugin-terser": "^7.0.2",
Expand Down
14 changes: 7 additions & 7 deletions rollup.config.js
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ function getConfig(entry, filename, perf) {
entryFileNames: filename,
chunkFileNames: `${basename}-[name]-[hash].js`,
format: 'esm',
exports: 'named'
exports: 'named',
},
plugins: [
!perf &&
Expand All @@ -24,18 +24,18 @@ function getConfig(entry, filename, perf) {
'perf-deets': path.resolve(
__dirname,
'./node_modules/perf-deets/noop.js'
)
}
),
},
}),
webWorkerLoader({
pattern: /.*\/worker\.js/,
targetPlatform: 'browser',
external: [],
plugins: [terser()]
plugins: [terser()],
}),
nodeResolve()
nodeResolve(),
],
...(perf ? { external: ['perf-deets'] } : {})
...(perf ? { external: ['perf-deets'] } : {}),
};
}

Expand All @@ -45,5 +45,5 @@ export default [
getConfig('src/indexeddb/backend.js', 'indexeddb-backend.js'),
getConfig('src/indexeddb/main-thread.js', 'indexeddb-main-thread.js'),
getConfig('src/indexeddb/backend.js', 'indexeddb-backend.js', true),
getConfig('src/indexeddb/main-thread.js', 'indexeddb-main-thread.js', true)
getConfig('src/indexeddb/main-thread.js', 'indexeddb-main-thread.js', true),
];
75 changes: 51 additions & 24 deletions src/examples/bench/index.html
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
<!DOCTYPE html>
<body>
<style type="text/css">
html, body, .container {
html,
body,
.container {
height: 100%;
}

Expand Down Expand Up @@ -43,7 +45,7 @@
display: block;
margin-bottom: 5px;
}

.output > div {
margin-bottom: 10px;
}
Expand Down Expand Up @@ -81,19 +83,20 @@

<div class="container">
<div class="text">
This is sqlite3 running in your browser with a backend that
properly persists the database in IndexedDB
(<a href="https://github.com/jlongster/absurd-sql">absurd-sql</a>).
It stores each page from the db as a separate item, treating IDB
like a hard disk. It never has to load the full DB into memory
and can update it with small individual writes.
This is sqlite3 running in your browser with a backend that properly
persists the database in IndexedDB (<a
href="https://github.com/jlongster/absurd-sql"
>absurd-sql</a
>). It stores each page from the db as a separate item, treating IDB like
a hard disk. It never has to load the full DB into memory and can update
it with small individual writes.
</div>

<div class="text last">
The below examples are meant to be stress tests, showing that it
can handle large amounts of data and queries that need to scan
all of it. With more normal cases and a small cache, it works
really well. It easily beats IndexedDB up to a factor of 10.
The below examples are meant to be stress tests, showing that it can
handle large amounts of data and queries that need to scan all of it. With
more normal cases and a small cache, it works really well. It easily beats
IndexedDB up to a factor of 10.
</div>

<div>
Expand All @@ -110,31 +113,55 @@
</div>

<div class="options">
<label><input type="checkbox" name="profile"> Record performance profile</label>
<label><input type="checkbox" name="raw-indexeddb"> Use raw IndexedDB</label>
<label
><input type="checkbox" name="profile" /> Record performance
profile</label
>
<label
><input type="checkbox" name="raw-indexeddb" /> Use raw IndexedDB</label
>
</div>

<div class="disable-if-raw-idb">
<div class="options">
Backend:
<label><input type="radio" name="backend" value="idb" checked> IndexedDB</label>
<label><input type="radio" name="backend" value="memory"> Memory</label>
<label
><input type="radio" name="backend" value="idb" checked />
IndexedDB</label
>
<label
><input type="radio" name="backend" value="memory" /> Memory</label
>
</div>

<div class="options">
Cache size:
<label><input type="radio" name="cacheSize" value="0" checked> 0MB</label>
<label><input type="radio" name="cacheSize" value="2000"> 2MB</label>
<label><input type="radio" name="cacheSize" value="10000"> 10MB</label>
<label><input type="radio" name="cacheSize" value="60000"> 60MB</label>
<span class="warning" style="font-size: 13px; color: #8080a0">Using a cache will greatly improve perf, but no cache shows the full number of read/writes</span>
<label
><input type="radio" name="cacheSize" value="0" checked /> 0MB</label
>
<label><input type="radio" name="cacheSize" value="2000" /> 2MB</label>
<label
><input type="radio" name="cacheSize" value="10000" /> 10MB</label
>
<label
><input type="radio" name="cacheSize" value="60000" /> 60MB</label
>
<span class="warning" style="font-size: 13px; color: #8080a0"
>Using a cache will greatly improve perf, but no cache shows the full
number of read/writes</span
>
</div>

<div class="options pageSize">
Page size:
<label><input type="radio" name="pageSize" value="4096" checked> 4096</label>
<label><input type="radio" name="pageSize" value="8192"> 8192</label>
<label><input type="radio" name="pageSize" value="16384"> 16384</label>
<label
><input type="radio" name="pageSize" value="4096" checked />
4096</label
>
<label><input type="radio" name="pageSize" value="8192" /> 8192</label>
<label
><input type="radio" name="pageSize" value="16384" /> 16384</label
>
</div>
</div>

Expand Down
20 changes: 10 additions & 10 deletions src/examples/bench/main.js
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ function init() {

worker.postMessage({ type: 'ui-invoke', name: 'init' });

worker.addEventListener('message', e => {
worker.addEventListener('message', (e) => {
switch (e.data.type) {
case 'output': {
output(e.data.msg);
Expand All @@ -59,7 +59,7 @@ function init() {
});

for (let input of document.querySelectorAll('input[type=radio]')) {
input.addEventListener('change', e => {
input.addEventListener('change', (e) => {
let name = e.target.name;
let value = e.target.value;
worker.postMessage({ type: 'options', name, value });
Expand All @@ -73,27 +73,27 @@ function init() {
document.querySelector('input[name="pageSize"][value="4096"]').checked = true;

let profile = document.querySelector('input[name="profile"]');
profile.addEventListener('click', e => {
profile.addEventListener('click', (e) => {
worker.postMessage({ type: 'profiling', on: e.target.checked });
});
worker.postMessage({ type: 'profiling', on: profile.checked });

let rawIDB = document.querySelector('input[name="raw-indexeddb"]');
rawIDB.addEventListener('click', e => {
rawIDB.addEventListener('click', (e) => {
document.querySelector('.disable-if-raw-idb').style.opacity = e.target
.checked
? 0.3
: 1;
worker.postMessage({
type: 'options',
name: 'raw-idb',
on: e.target.checked
on: e.target.checked,
});
});
worker.postMessage({
type: 'options',
name: 'raw-idb',
on: rawIDB.checked
on: rawIDB.checked,
});
}

Expand All @@ -105,7 +105,7 @@ let methods = [
'randomReads',
'deleteFile',
'readBench',
'writeBench'
'writeBench',
];

for (let method of methods) {
Expand All @@ -119,11 +119,11 @@ for (let method of methods) {

init();

window.runQuery = sql => {
window.runQuery = (sql) => {
let reqId = Math.random();

let promise = new Promise(resolve => {
let handler = e => {
let promise = new Promise((resolve) => {
let handler = (e) => {
if (e.data.type === 'query-results' && e.data.id === reqId) {
worker.removeEventListener('message', handler);
resolve(e.data.data);
Expand Down
Loading