Feat: Encryption with multiple ciphers (#23)

* wip: sqlite3mc

* random

* register vfs

* Split multi cipher builds

* Add changeset

* Export multicipher module

* Fix multiple cipher builds

* Test emcc flags

* Test mc sqlite

* WIP: Testing new builds

* Add xCurrentTimex64 implementation

* xCurrentTimeInt64 implementation

* Update emcc flags to normal builds

* Clean up

* Export non asyncify mc wasm

* Clean up demo logs

* PR feedback

* Revert formatting

* Revert formatting

* Revert stack size

* SImplify xcurrenttime

* Add julian epoch reference

---------

Co-authored-by: stevensJourney <[email protected]>

Author: mugikhan

.changeset/violet-laws-prove.md

@@ -0,0 +1,5 @@
+---
+"@journeyapps/wa-sqlite": minor
+---
+
+Enable database encryption with multiple ciphers

Makefile

@@ -1,7 +1,8 @@
 # dependencies
-SQLITE_VERSION = version-3.47.0
-SQLITE_TARBALL_URL = https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=${SQLITE_VERSION}
-
+SQLITE_VERSION = 3.47.0
+MC_SQLITE_VERSION = 1.9.0
+SQLITE_TARBALL_URL = https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=version-${SQLITE_VERSION}
+MC_SQLITE_URL = https://github.com/utelle/SQLite3MultipleCiphers/releases/download/v${MC_SQLITE_VERSION}/sqlite3mc-${MC_SQLITE_VERSION}-sqlite-${SQLITE_VERSION}-amalgamation.zip
 EXTENSION_FUNCTIONS = extension-functions.c
 EXTENSION_FUNCTIONS_URL = https://www.sqlite.org/contrib/download/extension-functions.c?get=25
 EXTENSION_FUNCTIONS_SHA3 = ee39ddf5eaa21e1d0ebcbceeab42822dd0c4f82d8039ce173fd4814807faabfa
@@ -21,6 +22,17 @@ CFILES = \
 
 POWERSYNC_CFILES = $(notdir $(wildcard powersync-static/*.c))
 
+MC_CFILES = \
+	sqlite3mc_amalgamation.c \
+	extension-functions.c \
+	main.c \
+	libauthorizer.c \
+	libfunction.c \
+	libhook.c \
+	libprogress.c \
+	libvfs.c \
+	$(CFILES_EXTRA)
+
 JSFILES = \
 	src/libauthorizer.js \
 	src/libfunction.js \
@@ -35,9 +47,11 @@ vpath %.c powersync-static
 
 POWERSYNC_EXPORTED_FUNCTIONS = powersync-static/powersync_exported_functions.json
 EXPORTED_FUNCTIONS = src/exported_functions.json
+MC_EXPORTED_FUNCTIONS = multiple-ciphers/mc_exported_functions.json
 
 # EMCC does not support multiple exports files. Need to combine them temporarily
 COMBINED_EXPORTED_FUNCTIONS = tmp/combined_exports.json
+MC_COMBINED_EXPORTED_FUNCTIONS = tmp/mc_combined_exports.json
 
 EXPORTED_RUNTIME_METHODS = src/extra_exported_runtime_methods.json
 ASYNCIFY_IMPORTS = src/asyncify_imports.json
@@ -51,6 +65,9 @@ POWERSYNC_OBJ_FILES_DEBUG = $(patsubst %.c,tmp/powersync-obj/debug/%.o,$(POWERSY
 POWERSYNC_OBJ_FILES_DIST = $(patsubst %.c,tmp/powersync-obj/dist/%.o,$(POWERSYNC_CFILES))
 POWERSYNC_STATIC_FILES = powersync-libs/libpowersync-wasm.a
 
+MC_OBJ_FILES_DEBUG = $(patsubst %.c,tmp/mc-obj/debug/%.o,$(MC_CFILES))
+MC_OBJ_FILES_DIST = $(patsubst %.c,tmp/mc-obj/dist/%.o,$(MC_CFILES))
+
 # build options
 EMCC ?= emcc
 
@@ -99,6 +116,10 @@ COMBINED_EMFLAGS_INTERFACES = \
 	-s EXPORTED_FUNCTIONS=@$(COMBINED_EXPORTED_FUNCTIONS) \
 	-s EXPORTED_RUNTIME_METHODS=@$(EXPORTED_RUNTIME_METHODS)
 
+MC_COMBINED_EMFLAGS_INTERFACES = \
+	-s EXPORTED_FUNCTIONS=@$(MC_COMBINED_EXPORTED_FUNCTIONS) \
+	-s EXPORTED_RUNTIME_METHODS=@$(EXPORTED_RUNTIME_METHODS)
+
 EMFLAGS_LIBRARIES = \
 	--js-library src/libadapters.js \
 	--post-js src/libauthorizer.js \
@@ -129,6 +150,7 @@ WASQLITE_DEFINES = \
 	-DSQLITE_DEFAULT_MEMSTATUS=0 \
 	-DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1 \
 	-DSQLITE_DQS=0 \
+	-D__WASM__ \
 	-DSQLITE_LIKE_DOESNT_MATCH_BLOBS \
 	-DSQLITE_MAX_EXPR_DEPTH=0 \
 	-DSQLITE_OMIT_AUTOINIT \
@@ -171,12 +193,23 @@ clean-deps:
 $(COMBINED_EXPORTED_FUNCTIONS): $(EXPORTED_FUNCTIONS) $(POWERSYNC_EXPORTED_FUNCTIONS)
 	jq -s 'add' $(EXPORTED_FUNCTIONS) $(POWERSYNC_EXPORTED_FUNCTIONS) > $(COMBINED_EXPORTED_FUNCTIONS)
 
+$(MC_COMBINED_EXPORTED_FUNCTIONS): $(EXPORTED_FUNCTIONS) $(POWERSYNC_EXPORTED_FUNCTIONS) $(MC_EXPORTED_FUNCTIONS)
+	jq -s 'add' $(EXPORTED_FUNCTIONS) $(POWERSYNC_EXPORTED_FUNCTIONS) $(MC_EXPORTED_FUNCTIONS) > $(MC_COMBINED_EXPORTED_FUNCTIONS)
+
 deps/$(SQLITE_VERSION)/sqlite3.h deps/$(SQLITE_VERSION)/sqlite3.c:
 	mkdir -p cache/$(SQLITE_VERSION)
 	curl -LsS $(SQLITE_TARBALL_URL) | tar -xzf - -C cache/$(SQLITE_VERSION)/ --strip-components=1
 	mkdir -p deps/$(SQLITE_VERSION)
 	(cd deps/$(SQLITE_VERSION); ../../cache/$(SQLITE_VERSION)/configure --enable-all && make sqlite3.c)
 
+# Download and extract sqlite3mc_amalgamation.c to the deps directory
+deps/$(SQLITE_VERSION)/sqlite3mc_amalgamation.c:
+	mkdir -p cache/sqlite3mc-$(MC_SQLITE_VERSION)
+	curl -LsS $(MC_SQLITE_URL) -o cache/sqlite3mc-$(MC_SQLITE_VERSION)/sqlite3mc.zip
+	unzip -o cache/sqlite3mc-$(MC_SQLITE_VERSION)/sqlite3mc.zip -d cache/sqlite3mc-$(MC_SQLITE_VERSION)
+	rm -rf cache/sqlite3mc-$(MC_SQLITE_VERSION)/sqlite3mc.zip
+	cp cache/sqlite3mc-$(MC_SQLITE_VERSION)/sqlite3mc_amalgamation.c deps/$(SQLITE_VERSION)/sqlite3mc_amalgamation.c
+
 # Download static files from PowerSync Core repository
 $(POWERSYNC_STATIC_FILES):
 	node scripts/download-core-build.js
@@ -241,14 +274,24 @@ tmp/powersync-obj/dist/%.o: %.c
 	mkdir -p tmp/powersync-obj/dist
 	$(EMCC) $(CFLAGS_DIST) $(WASQLITE_DEFINES) $^ -c -o $@
 
+# Build multiple ciphers
+tmp/mc-obj/debug/%.o: %.c
+	mkdir -p tmp/mc-obj/debug
+	$(EMCC) $(CFLAGS_DEBUG) $(WASQLITE_DEFINES) $^ -c -o $@
+
+# Build multiple ciphers
+tmp/mc-obj/dist/%.o: %.c
+	mkdir -p tmp/mc-obj/dist
+	$(EMCC) $(CFLAGS_DIST) $(WASQLITE_DEFINES) $^ -c -o $@
+
 
 ## debug
 .PHONY: clean-debug
 clean-debug:
 	rm -rf debug
 
 .PHONY: debug
-debug: debug/wa-sqlite.mjs debug/wa-sqlite-async.mjs debug/wa-sqlite-jspi.mjs debug/wa-sqlite-dynamic-main.mjs debug/wa-sqlite-dynamic-main-async.mjs
+debug: debug/wa-sqlite.mjs debug/wa-sqlite-async.mjs debug/wa-sqlite-jspi.mjs debug/wa-sqlite-dynamic-main.mjs debug/wa-sqlite-dynamic-main-async.mjs debug/mc-wa-sqlite.mjs debug/mc-wa-sqlite-async.mjs debug/mc-wa-sqlite-jspi.mjs
 
 # Statically links PowerSync Core
 debug/wa-sqlite.mjs: $(OBJ_FILES_DEBUG) $(POWERSYNC_OBJ_FILES_DEBUG) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(POWERSYNC_STATIC_FILES) $(COMBINED_EXPORTED_FUNCTIONS)
@@ -261,7 +304,7 @@ debug/wa-sqlite.mjs: $(OBJ_FILES_DEBUG) $(POWERSYNC_OBJ_FILES_DEBUG) $(EXPORTED_
 	  $(POWERSYNC_OBJ_FILES_DEBUG) -o $@
 
 # Statically links PowerSync Core
-debug/wa-sqlite-async.mjs: $(OBJ_FILES_DEBUG) $(POWERSYNC_OBJ_FILES_DEBUG)  $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS) $(POWERSYNC_STATIC_FILES)  $(COMBINED_EXPORTED_FUNCTIONS)
+debug/wa-sqlite-async.mjs: $(OBJ_FILES_DEBUG) $(POWERSYNC_OBJ_FILES_DEBUG) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS) $(POWERSYNC_STATIC_FILES) $(COMBINED_EXPORTED_FUNCTIONS)
 	mkdir -p debug
 	$(EMCC) $(EMFLAGS_DEBUG) \
 	  $(COMBINED_EMFLAGS_INTERFACES) \
@@ -272,7 +315,7 @@ debug/wa-sqlite-async.mjs: $(OBJ_FILES_DEBUG) $(POWERSYNC_OBJ_FILES_DEBUG)  $(EX
 	  $(OBJ_FILES_DEBUG) -o $@
 
 # Statically links PowerSync Core
-debug/wa-sqlite-jspi.mjs: $(OBJ_FILES_DEBUG) $(POWERSYNC_OBJ_FILES_DEBUG) $(JSFILES) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS) $(POWERSYNC_STATIC_FILES)  $(COMBINED_EXPORTED_FUNCTIONS)
+debug/wa-sqlite-jspi.mjs: $(OBJ_FILES_DEBUG) $(POWERSYNC_OBJ_FILES_DEBUG) $(JSFILES) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS) $(POWERSYNC_STATIC_FILES) $(COMBINED_EXPORTED_FUNCTIONS)
 	mkdir -p debug
 	$(EMCC) $(EMFLAGS_DEBUG) $(EMFLAGS_DYNAMIC)  \
 	  $(COMBINED_EMFLAGS_INTERFACES) \
@@ -282,6 +325,38 @@ debug/wa-sqlite-jspi.mjs: $(OBJ_FILES_DEBUG) $(POWERSYNC_OBJ_FILES_DEBUG) $(JSFI
 	  $(POWERSYNC_OBJ_FILES_DEBUG) \
 	  $(OBJ_FILES_DEBUG) -o $@
 
+# Statically links PowerSync Core
+debug/mc-wa-sqlite.mjs: $(MC_OBJ_FILES_DEBUG) $(POWERSYNC_OBJ_FILES_DEBUG) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(POWERSYNC_STATIC_FILES) $(MC_COMBINED_EXPORTED_FUNCTIONS)
+	mkdir -p debug
+	$(EMCC) $(EMFLAGS_DEBUG) \
+	  $(COMBINED_EMFLAGS_INTERFACES) \
+	  $(EMFLAGS_LIBRARIES) \
+	  $(POWERSYNC_STATIC_FILES) \
+	  $(OBJ_FILES_DEBUG) \
+	  $(MC_OBJ_FILES_DEBUG) -o $@
+
+# Statically links PowerSync Core
+debug/mc-wa-sqlite-async.mjs: $(MC_OBJ_FILES_DEBUG) $(POWERSYNC_OBJ_FILES_DEBUG)  $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS) $(POWERSYNC_STATIC_FILES) $(MC_COMBINED_EXPORTED_FUNCTIONS)
+	mkdir -p debug
+	$(EMCC) $(EMFLAGS_DEBUG) \
+	  $(COMBINED_EMFLAGS_INTERFACES) \
+	  $(EMFLAGS_LIBRARIES) \
+	  $(EMFLAGS_ASYNCIFY_DEBUG) \
+	  $(POWERSYNC_STATIC_FILES) \
+	  $(POWERSYNC_OBJ_FILES_DEBUG) \
+	  $(MC_OBJ_FILES_DEBUG) -o $@
+
+# Statically links PowerSync Core
+debug/mc-wa-sqlite-jspi.mjs: $(MC_OBJ_FILES_DEBUG) $(POWERSYNC_OBJ_FILES_DEBUG) $(JSFILES) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS) $(POWERSYNC_STATIC_FILES) $(MC_COMBINED_EXPORTED_FUNCTIONS)
+	mkdir -p debug
+	$(EMCC) $(EMFLAGS_DEBUG) $(EMFLAGS_DYNAMIC)  \
+	  $(COMBINED_EMFLAGS_INTERFACES) \
+	  $(EMFLAGS_LIBRARIES) \
+	  $(EMFLAGS_JSPI) \
+	  $(POWERSYNC_STATIC_FILES) \
+	  $(POWERSYNC_OBJ_FILES_DEBUG) \
+	  $(MC_OBJ_FILES_DEBUG) -o $@
+
 
 # Dynamic main module
 # Exported functions are omitted here since everything is currently exported
@@ -330,10 +405,10 @@ clean-dist:
 	rm -rf dist
 
 .PHONY: dist
-dist: dist/wa-sqlite.mjs dist/wa-sqlite-async.mjs dist/wa-sqlite-jspi.mjs dist/wa-sqlite-dynamic-main.mjs dist/wa-sqlite-async-dynamic-main.mjs
+dist: dist/wa-sqlite.mjs dist/wa-sqlite-async.mjs dist/wa-sqlite-jspi.mjs dist/wa-sqlite-dynamic-main.mjs dist/wa-sqlite-async-dynamic-main.mjs dist/mc-wa-sqlite.mjs dist/mc-wa-sqlite-async.mjs dist/mc-wa-sqlite-jspi.mjs
 
 # Statically links PowerSync Core
-dist/wa-sqlite.mjs: $(OBJ_FILES_DIST) $(POWERSYNC_OBJ_FILES_DIST) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(POWERSYNC_STATIC_FILES)  $(COMBINED_EXPORTED_FUNCTIONS)
+dist/wa-sqlite.mjs: $(OBJ_FILES_DIST) $(POWERSYNC_OBJ_FILES_DIST) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(POWERSYNC_STATIC_FILES) $(COMBINED_EXPORTED_FUNCTIONS)
 	mkdir -p dist
 	$(EMCC) $(EMFLAGS_DIST) \
 	  $(COMBINED_EMFLAGS_INTERFACES) \
@@ -343,7 +418,7 @@ dist/wa-sqlite.mjs: $(OBJ_FILES_DIST) $(POWERSYNC_OBJ_FILES_DIST) $(EXPORTED_FUN
 	  $(OBJ_FILES_DIST)  -o $@
 
 # Statically links PowerSync Core
-dist/wa-sqlite-async.mjs: $(OBJ_FILES_DIST) $(POWERSYNC_OBJ_FILES_DIST) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS) $(POWERSYNC_STATIC_FILES)  $(COMBINED_EXPORTED_FUNCTIONS)
+dist/wa-sqlite-async.mjs: $(OBJ_FILES_DIST) $(POWERSYNC_OBJ_FILES_DIST) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS) $(POWERSYNC_STATIC_FILES) $(COMBINED_EXPORTED_FUNCTIONS)
 	mkdir -p dist
 	$(EMCC) $(EMFLAGS_DIST) \
 	  $(COMBINED_EMFLAGS_INTERFACES) \
@@ -354,7 +429,7 @@ dist/wa-sqlite-async.mjs: $(OBJ_FILES_DIST) $(POWERSYNC_OBJ_FILES_DIST) $(EXPORT
 	  $(OBJ_FILES_DIST) -o $@
 
 # Statically links PowerSync Core
-dist/wa-sqlite-jspi.mjs: $(OBJ_FILES_DIST) $(POWERSYNC_OBJ_FILES_DIST) $(JSFILES) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS) $(POWERSYNC_STATIC_FILES)  $(COMBINED_EXPORTED_FUNCTIONS)
+dist/wa-sqlite-jspi.mjs: $(OBJ_FILES_DIST) $(POWERSYNC_OBJ_FILES_DIST) $(JSFILES) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS) $(POWERSYNC_STATIC_FILES) $(COMBINED_EXPORTED_FUNCTIONS)
 	mkdir -p dist
 	$(EMCC) $(EMFLAGS_DIST) \
 	  $(COMBINED_EMFLAGS_INTERFACES) \
@@ -364,6 +439,38 @@ dist/wa-sqlite-jspi.mjs: $(OBJ_FILES_DIST) $(POWERSYNC_OBJ_FILES_DIST) $(JSFILES
 	  $(POWERSYNC_OBJ_FILES_DIST) \
 	  $(OBJ_FILES_DIST) -o $@
 
+# Statically links PowerSync Core with multiple ciphers
+dist/mc-wa-sqlite.mjs: $(MC_OBJ_FILES_DIST) $(POWERSYNC_OBJ_FILES_DIST) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(POWERSYNC_STATIC_FILES) $(MC_COMBINED_EXPORTED_FUNCTIONS)
+	mkdir -p dist
+	$(EMCC) $(EMFLAGS_DIST) \
+	  $(MC_COMBINED_EMFLAGS_INTERFACES) \
+	  $(EMFLAGS_LIBRARIES) \
+	  $(POWERSYNC_STATIC_FILES) \
+	  $(POWERSYNC_OBJ_FILES_DIST) \
+	  $(MC_OBJ_FILES_DIST)  -o $@
+
+# Statically links PowerSync Core with multiple ciphers
+dist/mc-wa-sqlite-async.mjs: $(MC_OBJ_FILES_DIST) $(POWERSYNC_OBJ_FILES_DIST) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS) $(POWERSYNC_STATIC_FILES) $(MC_COMBINED_EXPORTED_FUNCTIONS)
+	mkdir -p dist
+	$(EMCC) $(EMFLAGS_DIST) \
+	  $(MC_COMBINED_EMFLAGS_INTERFACES) \
+	  $(EMFLAGS_LIBRARIES) \
+	  $(EMFLAGS_ASYNCIFY_DIST) \
+	  $(POWERSYNC_STATIC_FILES) \
+	  $(POWERSYNC_OBJ_FILES_DIST) \
+	  $(MC_OBJ_FILES_DIST) -o $@
+
+# Statically links PowerSync Core with multiple ciphers
+dist/mc-wa-sqlite-jspi.mjs: $(MC_OBJ_FILES_DIST) $(POWERSYNC_OBJ_FILES_DIST) $(JSFILES) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS) $(POWERSYNC_STATIC_FILES) $(MC_COMBINED_EXPORTED_FUNCTIONS)
+	mkdir -p dist
+	$(EMCC) $(EMFLAGS_DIST) \
+	  $(MC_COMBINED_EMFLAGS_INTERFACES) \
+	  $(EMFLAGS_LIBRARIES) \
+	  $(EMFLAGS_JSPI) \
+	  $(POWERSYNC_STATIC_FILES) \
+	  $(POWERSYNC_OBJ_FILES_DIST) \
+	  $(MC_OBJ_FILES_DIST) -o $@
+
 # Dynamic main module
 # Exported functions are omitted here since everything is currently exported
 dist/wa-sqlite-dynamic-main.mjs: $(OBJ_FILES_DIST) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS)  $(COMBINED_EXPORTED_FUNCTIONS)

demo/demo-worker.js

@@ -4,8 +4,11 @@ import * as SQLite from '../src/sqlite-api.js';
 
 const BUILDS = new Map([
   ['default', '../dist/wa-sqlite.mjs'],
+  ['mc-default', '../dist/mc-wa-sqlite.mjs'],
   ['asyncify', '../dist/wa-sqlite-async.mjs'],
+  ['mc-asyncify', '../dist/mc-wa-sqlite-async.mjs'],
   ['jspi', '../dist/wa-sqlite-jspi.mjs'],
+  ['mc-jspi', '../dist/mc-wa-sqlite-jspi.mjs'],
   // ['default', '../debug/wa-sqlite.mjs'],
   // ['asyncify', '../debug/wa-sqlite-async.mjs'],
   // ['jspi', '../debug/wa-sqlite-jspi.mjs'],
@@ -103,7 +106,6 @@ maybeReset().then(async () => {
 
   const sqlite3 = SQLite.Factory(module);
 
-
   if (buildName.endsWith('-dynamic')) {
     const extWasm = EXT_WASM.get(buildName);
     // Load the extension library into this scope
@@ -130,6 +132,12 @@ maybeReset().then(async () => {
     sqlite3.vfs_register(vfs, true);
   }
 
+  if(buildName.startsWith('mc-')) {
+    const createResult = module.ccall('sqlite3mc_vfs_create', 'int', ['string', 'int'], [vfsName, 1]);
+    if (createResult !== 0) {
+      throw new Error(`sqlite3mc_vfs_create failed with ${createResult}`);
+    }
+  }
   // Open the database.
   const db = await sqlite3.open_v2(dbName);