Skip to content

Feat: Encryption with multiple ciphers #23

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 22 commits into from
Dec 10, 2024
Merged

Feat: Encryption with multiple ciphers #23

Show file tree
Hide file tree
Changes from 16 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
8c1c14a
wip: sqlite3mc
stevensJourney Nov 19, 2024
c645b95
random
stevensJourney Nov 20, 2024
6437b19
register vfs
stevensJourney Nov 20, 2024
38cc5c3
Split multi cipher builds
mugikhan Nov 20, 2024
6a7cee4
Add changeset
mugikhan Nov 21, 2024
e3c2c05
Export multicipher module
mugikhan Nov 21, 2024
7c5f319
Fix multiple cipher builds
mugikhan Nov 21, 2024
6fea237
Test emcc flags
mugikhan Nov 22, 2024
99c5df8
Test mc sqlite
mugikhan Nov 25, 2024
199bf09
WIP: Testing new builds
mugikhan Nov 26, 2024
56319e3
Add xCurrentTimex64 implementation
mugikhan Nov 26, 2024
bc8a9c1
xCurrentTimeInt64 implementation
mugikhan Nov 26, 2024
1eb90af
Update emcc flags to normal builds
mugikhan Nov 26, 2024
c0c2108
Clean up
mugikhan Nov 27, 2024
d1f8854
Export non asyncify mc wasm
mugikhan Nov 27, 2024
ae0c3b4
Clean up demo logs
mugikhan Nov 27, 2024
0021eac
PR feedback
mugikhan Nov 27, 2024
9113d9c
Revert formatting
mugikhan Nov 28, 2024
96cb6d9
Revert formatting
mugikhan Nov 28, 2024
94e4b06
Revert stack size
mugikhan Nov 29, 2024
7fd92da
SImplify xcurrenttime
mugikhan Dec 2, 2024
2da0c82
Add julian epoch reference
mugikhan Dec 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/violet-laws-prove.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"@journeyapps/wa-sqlite": minor
---

Enable database encryption with multiple ciphers
130 changes: 120 additions & 10 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
# dependencies
SQLITE_VERSION = version-3.47.0
MC_SQLITE_VERSION = 1.9.0
SQLITE_TARBALL_URL = https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=${SQLITE_VERSION}

MC_SQLITE_URL = https://github.com/utelle/SQLite3MultipleCiphers/releases/download/v${MC_SQLITE_VERSION}/sqlite3mc-${MC_SQLITE_VERSION}-sqlite-3.47.0-amalgamation.zip
EXTENSION_FUNCTIONS = extension-functions.c
EXTENSION_FUNCTIONS_URL = https://www.sqlite.org/contrib/download/extension-functions.c?get=25
EXTENSION_FUNCTIONS_SHA3 = ee39ddf5eaa21e1d0ebcbceeab42822dd0c4f82d8039ce173fd4814807faabfa
Expand All @@ -21,6 +22,17 @@ CFILES = \

POWERSYNC_CFILES = $(notdir $(wildcard powersync-static/*.c))

MC_CFILES = \
sqlite3mc_amalgamation.c \
extension-functions.c \
main.c \
libauthorizer.c \
libfunction.c \
libhook.c \
libprogress.c \
libvfs.c \
$(CFILES_EXTRA)

JSFILES = \
src/libauthorizer.js \
src/libfunction.js \
Expand All @@ -35,9 +47,11 @@ vpath %.c powersync-static

POWERSYNC_EXPORTED_FUNCTIONS = powersync-static/powersync_exported_functions.json
EXPORTED_FUNCTIONS = src/exported_functions.json
MC_EXPORTED_FUNCTIONS = multiple-ciphers/mc_exported_functions.json

# EMCC does not support multiple exports files. Need to combine them temporarily
COMBINED_EXPORTED_FUNCTIONS = tmp/combined_exports.json
MC_COMBINED_EXPORTED_FUNCTIONS = tmp/mc_combined_exports.json

EXPORTED_RUNTIME_METHODS = src/extra_exported_runtime_methods.json
ASYNCIFY_IMPORTS = src/asyncify_imports.json
Expand All @@ -51,6 +65,9 @@ POWERSYNC_OBJ_FILES_DEBUG = $(patsubst %.c,tmp/powersync-obj/debug/%.o,$(POWERSY
POWERSYNC_OBJ_FILES_DIST = $(patsubst %.c,tmp/powersync-obj/dist/%.o,$(POWERSYNC_CFILES))
POWERSYNC_STATIC_FILES = powersync-libs/libpowersync-wasm.a

MC_OBJ_FILES_DEBUG = $(patsubst %.c,tmp/mc-obj/debug/%.o,$(MC_CFILES))
MC_OBJ_FILES_DIST = $(patsubst %.c,tmp/mc-obj/dist/%.o,$(MC_CFILES))

# build options
EMCC ?= emcc

Expand Down Expand Up @@ -99,6 +116,10 @@ COMBINED_EMFLAGS_INTERFACES = \
-s EXPORTED_FUNCTIONS=@$(COMBINED_EXPORTED_FUNCTIONS) \
-s EXPORTED_RUNTIME_METHODS=@$(EXPORTED_RUNTIME_METHODS)

MC_COMBINED_EMFLAGS_INTERFACES = \
-s EXPORTED_FUNCTIONS=@$(MC_COMBINED_EXPORTED_FUNCTIONS) \
-s EXPORTED_RUNTIME_METHODS=@$(EXPORTED_RUNTIME_METHODS)

EMFLAGS_LIBRARIES = \
--js-library src/libadapters.js \
--post-js src/libauthorizer.js \
Expand All @@ -113,11 +134,14 @@ EMFLAGS_ASYNCIFY_COMMON = \

EMFLAGS_ASYNCIFY_DEBUG = \
$(EMFLAGS_ASYNCIFY_COMMON) \
-s ASYNCIFY_STACK_SIZE=24576
-s ASYNCIFY_STACK_SIZE=49152
# TODO sensible stack size increase

EMFLAGS_ASYNCIFY_DIST = \
$(EMFLAGS_ASYNCIFY_COMMON) \
-s ASYNCIFY_STACK_SIZE=16384
-s ASYNCIFY_STACK_SIZE=32768
# TODO sensible stack size increase


EMFLAGS_JSPI = \
-s JSPI \
Expand All @@ -129,6 +153,7 @@ WASQLITE_DEFINES = \
-DSQLITE_DEFAULT_MEMSTATUS=0 \
-DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1 \
-DSQLITE_DQS=0 \
-D__WASM__ \
-DSQLITE_LIKE_DOESNT_MATCH_BLOBS \
-DSQLITE_MAX_EXPR_DEPTH=0 \
-DSQLITE_OMIT_AUTOINIT \
Expand Down Expand Up @@ -171,12 +196,23 @@ clean-deps:
$(COMBINED_EXPORTED_FUNCTIONS): $(EXPORTED_FUNCTIONS) $(POWERSYNC_EXPORTED_FUNCTIONS)
jq -s 'add' $(EXPORTED_FUNCTIONS) $(POWERSYNC_EXPORTED_FUNCTIONS) > $(COMBINED_EXPORTED_FUNCTIONS)

$(MC_COMBINED_EXPORTED_FUNCTIONS): $(EXPORTED_FUNCTIONS) $(POWERSYNC_EXPORTED_FUNCTIONS) $(MC_EXPORTED_FUNCTIONS)
jq -s 'add' $(EXPORTED_FUNCTIONS) $(POWERSYNC_EXPORTED_FUNCTIONS) $(MC_EXPORTED_FUNCTIONS) > $(MC_COMBINED_EXPORTED_FUNCTIONS)

deps/$(SQLITE_VERSION)/sqlite3.h deps/$(SQLITE_VERSION)/sqlite3.c:
mkdir -p cache/$(SQLITE_VERSION)
curl -LsS $(SQLITE_TARBALL_URL) | tar -xzf - -C cache/$(SQLITE_VERSION)/ --strip-components=1
mkdir -p deps/$(SQLITE_VERSION)
(cd deps/$(SQLITE_VERSION); ../../cache/$(SQLITE_VERSION)/configure --enable-all && make sqlite3.c)

# Download and extract sqlite3mc_amalgamation.c to the deps directory
deps/$(SQLITE_VERSION)/sqlite3mc_amalgamation.c:
mkdir -p cache/sqlite3mc-$(MC_SQLITE_VERSION)
curl -LsS $(MC_SQLITE_URL) -o cache/sqlite3mc-$(MC_SQLITE_VERSION)/sqlite3mc.zip
unzip -o cache/sqlite3mc-$(MC_SQLITE_VERSION)/sqlite3mc.zip -d cache/sqlite3mc-$(MC_SQLITE_VERSION)
rm -rf cache/sqlite3mc-$(MC_SQLITE_VERSION)/sqlite3mc.zip
cp cache/sqlite3mc-$(MC_SQLITE_VERSION)/sqlite3mc_amalgamation.c deps/$(SQLITE_VERSION)/sqlite3mc_amalgamation.c

# Download static files from PowerSync Core repository
$(POWERSYNC_STATIC_FILES):
node scripts/download-core-build.js
Expand Down Expand Up @@ -241,14 +277,24 @@ tmp/powersync-obj/dist/%.o: %.c
mkdir -p tmp/powersync-obj/dist
$(EMCC) $(CFLAGS_DIST) $(WASQLITE_DEFINES) $^ -c -o $@

# Build multiple ciphers
tmp/mc-obj/debug/%.o: %.c
mkdir -p tmp/mc-obj/debug
$(EMCC) $(CFLAGS_DEBUG) $(WASQLITE_DEFINES) $^ -c -o $@

# Build multiple ciphers
tmp/mc-obj/dist/%.o: %.c
mkdir -p tmp/mc-obj/dist
$(EMCC) $(CFLAGS_DIST) $(WASQLITE_DEFINES) $^ -c -o $@


## debug
.PHONY: clean-debug
clean-debug:
rm -rf debug

.PHONY: debug
debug: debug/wa-sqlite.mjs debug/wa-sqlite-async.mjs debug/wa-sqlite-jspi.mjs debug/wa-sqlite-dynamic-main.mjs debug/wa-sqlite-dynamic-main-async.mjs
debug: debug/wa-sqlite.mjs debug/wa-sqlite-async.mjs debug/wa-sqlite-jspi.mjs debug/wa-sqlite-dynamic-main.mjs debug/wa-sqlite-dynamic-main-async.mjs debug/mc-wa-sqlite.mjs debug/mc-wa-sqlite-async.mjs debug/mc-wa-sqlite-jspi.mjs

# Statically links PowerSync Core
debug/wa-sqlite.mjs: $(OBJ_FILES_DEBUG) $(POWERSYNC_OBJ_FILES_DEBUG) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(POWERSYNC_STATIC_FILES) $(COMBINED_EXPORTED_FUNCTIONS)
Expand All @@ -261,7 +307,7 @@ debug/wa-sqlite.mjs: $(OBJ_FILES_DEBUG) $(POWERSYNC_OBJ_FILES_DEBUG) $(EXPORTED_
$(POWERSYNC_OBJ_FILES_DEBUG) -o $@

# Statically links PowerSync Core
debug/wa-sqlite-async.mjs: $(OBJ_FILES_DEBUG) $(POWERSYNC_OBJ_FILES_DEBUG) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS) $(POWERSYNC_STATIC_FILES) $(COMBINED_EXPORTED_FUNCTIONS)
debug/wa-sqlite-async.mjs: $(OBJ_FILES_DEBUG) $(POWERSYNC_OBJ_FILES_DEBUG) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS) $(POWERSYNC_STATIC_FILES) $(COMBINED_EXPORTED_FUNCTIONS)
mkdir -p debug
$(EMCC) $(EMFLAGS_DEBUG) \
$(COMBINED_EMFLAGS_INTERFACES) \
Expand All @@ -272,7 +318,7 @@ debug/wa-sqlite-async.mjs: $(OBJ_FILES_DEBUG) $(POWERSYNC_OBJ_FILES_DEBUG) $(EX
$(OBJ_FILES_DEBUG) -o $@

# Statically links PowerSync Core
debug/wa-sqlite-jspi.mjs: $(OBJ_FILES_DEBUG) $(POWERSYNC_OBJ_FILES_DEBUG) $(JSFILES) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS) $(POWERSYNC_STATIC_FILES) $(COMBINED_EXPORTED_FUNCTIONS)
debug/wa-sqlite-jspi.mjs: $(OBJ_FILES_DEBUG) $(POWERSYNC_OBJ_FILES_DEBUG) $(JSFILES) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS) $(POWERSYNC_STATIC_FILES) $(COMBINED_EXPORTED_FUNCTIONS)
mkdir -p debug
$(EMCC) $(EMFLAGS_DEBUG) $(EMFLAGS_DYNAMIC) \
$(COMBINED_EMFLAGS_INTERFACES) \
Expand All @@ -282,6 +328,38 @@ debug/wa-sqlite-jspi.mjs: $(OBJ_FILES_DEBUG) $(POWERSYNC_OBJ_FILES_DEBUG) $(JSFI
$(POWERSYNC_OBJ_FILES_DEBUG) \
$(OBJ_FILES_DEBUG) -o $@

# Statically links PowerSync Core
debug/mc-wa-sqlite.mjs: $(MC_OBJ_FILES_DEBUG) $(POWERSYNC_OBJ_FILES_DEBUG) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(POWERSYNC_STATIC_FILES) $(MC_COMBINED_EXPORTED_FUNCTIONS)
mkdir -p debug
$(EMCC) $(EMFLAGS_DEBUG) \
$(COMBINED_EMFLAGS_INTERFACES) \
$(EMFLAGS_LIBRARIES) \
$(POWERSYNC_STATIC_FILES) \
$(OBJ_FILES_DEBUG) \
$(MC_OBJ_FILES_DEBUG) -o $@

# Statically links PowerSync Core
debug/mc-wa-sqlite-async.mjs: $(MC_OBJ_FILES_DEBUG) $(POWERSYNC_OBJ_FILES_DEBUG) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS) $(POWERSYNC_STATIC_FILES) $(MC_COMBINED_EXPORTED_FUNCTIONS)
mkdir -p debug
$(EMCC) $(EMFLAGS_DEBUG) \
$(COMBINED_EMFLAGS_INTERFACES) \
$(EMFLAGS_LIBRARIES) \
$(EMFLAGS_ASYNCIFY_DEBUG) \
$(POWERSYNC_STATIC_FILES) \
$(POWERSYNC_OBJ_FILES_DEBUG) \
$(MC_OBJ_FILES_DEBUG) -o $@

# Statically links PowerSync Core
debug/mc-wa-sqlite-jspi.mjs: $(MC_OBJ_FILES_DEBUG) $(POWERSYNC_OBJ_FILES_DEBUG) $(JSFILES) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS) $(POWERSYNC_STATIC_FILES) $(MC_COMBINED_EXPORTED_FUNCTIONS)
mkdir -p debug
$(EMCC) $(EMFLAGS_DEBUG) $(EMFLAGS_DYNAMIC) \
$(COMBINED_EMFLAGS_INTERFACES) \
$(EMFLAGS_LIBRARIES) \
$(EMFLAGS_JSPI) \
$(POWERSYNC_STATIC_FILES) \
$(POWERSYNC_OBJ_FILES_DEBUG) \
$(MC_OBJ_FILES_DEBUG) -o $@


# Dynamic main module
# Exported functions are omitted here since everything is currently exported
Expand Down Expand Up @@ -330,10 +408,10 @@ clean-dist:
rm -rf dist

.PHONY: dist
dist: dist/wa-sqlite.mjs dist/wa-sqlite-async.mjs dist/wa-sqlite-jspi.mjs dist/wa-sqlite-dynamic-main.mjs dist/wa-sqlite-async-dynamic-main.mjs
dist: dist/wa-sqlite.mjs dist/wa-sqlite-async.mjs dist/wa-sqlite-jspi.mjs dist/wa-sqlite-dynamic-main.mjs dist/wa-sqlite-async-dynamic-main.mjs dist/mc-wa-sqlite.mjs dist/mc-wa-sqlite-async.mjs dist/mc-wa-sqlite-jspi.mjs

# Statically links PowerSync Core
dist/wa-sqlite.mjs: $(OBJ_FILES_DIST) $(POWERSYNC_OBJ_FILES_DIST) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(POWERSYNC_STATIC_FILES) $(COMBINED_EXPORTED_FUNCTIONS)
dist/wa-sqlite.mjs: $(OBJ_FILES_DIST) $(POWERSYNC_OBJ_FILES_DIST) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(POWERSYNC_STATIC_FILES) $(COMBINED_EXPORTED_FUNCTIONS)
mkdir -p dist
$(EMCC) $(EMFLAGS_DIST) \
$(COMBINED_EMFLAGS_INTERFACES) \
Expand All @@ -343,7 +421,7 @@ dist/wa-sqlite.mjs: $(OBJ_FILES_DIST) $(POWERSYNC_OBJ_FILES_DIST) $(EXPORTED_FUN
$(OBJ_FILES_DIST) -o $@

# Statically links PowerSync Core
dist/wa-sqlite-async.mjs: $(OBJ_FILES_DIST) $(POWERSYNC_OBJ_FILES_DIST) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS) $(POWERSYNC_STATIC_FILES) $(COMBINED_EXPORTED_FUNCTIONS)
dist/wa-sqlite-async.mjs: $(OBJ_FILES_DIST) $(POWERSYNC_OBJ_FILES_DIST) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS) $(POWERSYNC_STATIC_FILES) $(COMBINED_EXPORTED_FUNCTIONS)
mkdir -p dist
$(EMCC) $(EMFLAGS_DIST) \
$(COMBINED_EMFLAGS_INTERFACES) \
Expand All @@ -354,7 +432,7 @@ dist/wa-sqlite-async.mjs: $(OBJ_FILES_DIST) $(POWERSYNC_OBJ_FILES_DIST) $(EXPORT
$(OBJ_FILES_DIST) -o $@

# Statically links PowerSync Core
dist/wa-sqlite-jspi.mjs: $(OBJ_FILES_DIST) $(POWERSYNC_OBJ_FILES_DIST) $(JSFILES) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS) $(POWERSYNC_STATIC_FILES) $(COMBINED_EXPORTED_FUNCTIONS)
dist/wa-sqlite-jspi.mjs: $(OBJ_FILES_DIST) $(POWERSYNC_OBJ_FILES_DIST) $(JSFILES) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS) $(POWERSYNC_STATIC_FILES) $(COMBINED_EXPORTED_FUNCTIONS)
mkdir -p dist
$(EMCC) $(EMFLAGS_DIST) \
$(COMBINED_EMFLAGS_INTERFACES) \
Expand All @@ -364,6 +442,38 @@ dist/wa-sqlite-jspi.mjs: $(OBJ_FILES_DIST) $(POWERSYNC_OBJ_FILES_DIST) $(JSFILES
$(POWERSYNC_OBJ_FILES_DIST) \
$(OBJ_FILES_DIST) -o $@

# Statically links PowerSync Core with multiple ciphers
dist/mc-wa-sqlite.mjs: $(MC_OBJ_FILES_DIST) $(POWERSYNC_OBJ_FILES_DIST) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(POWERSYNC_STATIC_FILES) $(MC_COMBINED_EXPORTED_FUNCTIONS)
mkdir -p dist
$(EMCC) $(EMFLAGS_DIST) \
$(MC_COMBINED_EMFLAGS_INTERFACES) \
$(EMFLAGS_LIBRARIES) \
$(POWERSYNC_STATIC_FILES) \
$(POWERSYNC_OBJ_FILES_DIST) \
$(MC_OBJ_FILES_DIST) -o $@

# Statically links PowerSync Core with multiple ciphers
dist/mc-wa-sqlite-async.mjs: $(MC_OBJ_FILES_DIST) $(POWERSYNC_OBJ_FILES_DIST) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS) $(POWERSYNC_STATIC_FILES) $(MC_COMBINED_EXPORTED_FUNCTIONS)
mkdir -p dist
$(EMCC) $(EMFLAGS_DIST) \
$(MC_COMBINED_EMFLAGS_INTERFACES) \
$(EMFLAGS_LIBRARIES) \
$(EMFLAGS_ASYNCIFY_DIST) \
$(POWERSYNC_STATIC_FILES) \
$(POWERSYNC_OBJ_FILES_DIST) \
$(MC_OBJ_FILES_DIST) -o $@

# Statically links PowerSync Core with multiple ciphers
dist/mc-wa-sqlite-jspi.mjs: $(MC_OBJ_FILES_DIST) $(POWERSYNC_OBJ_FILES_DIST) $(JSFILES) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS) $(POWERSYNC_STATIC_FILES) $(MC_COMBINED_EXPORTED_FUNCTIONS)
mkdir -p dist
$(EMCC) $(EMFLAGS_DIST) \
$(MC_COMBINED_EMFLAGS_INTERFACES) \
$(EMFLAGS_LIBRARIES) \
$(EMFLAGS_JSPI) \
$(POWERSYNC_STATIC_FILES) \
$(POWERSYNC_OBJ_FILES_DIST) \
$(MC_OBJ_FILES_DIST) -o $@

# Dynamic main module
# Exported functions are omitted here since everything is currently exported
dist/wa-sqlite-dynamic-main.mjs: $(OBJ_FILES_DIST) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(COMBINED_EXPORTED_FUNCTIONS)
Expand Down
Loading