Merge pull request #5 from sqliteai/wasm-build

add cloudsync-wasm build

Author: marcobambini

.github/workflows/main.yml

@@ -11,7 +11,7 @@ permissions:
 jobs:
   build:
     runs-on: ${{ matrix.os }}
-    name: ${{ matrix.name }}${{ matrix.arch && format('-{0}', matrix.arch) || '' }} build${{ matrix.arch != 'arm64-v8a' && matrix.name != 'isim' && matrix.name != 'ios' && ' + test' || ''}}
+    name: ${{ matrix.name }}${{ matrix.arch && format('-{0}', matrix.arch) || '' }} build${{ matrix.arch != 'arm64-v8a' && matrix.name != 'isim' && matrix.name != 'ios' && matrix.name != 'wasm' && ' + test' || ''}}
     timeout-minutes: 20
     strategy:
       fail-fast: false
@@ -43,6 +43,9 @@ jobs:
           - os: macos-latest
             name: isim
             make: PLATFORM=isim
+          - os: ubuntu-latest
+            name: wasm
+            make: PLATFORM=wasm
 
     defaults:
       run:
@@ -70,6 +73,10 @@ jobs:
         run: make curl/windows/libcurl.a
         shell: msys2 {0}
 
+      - name: wasm install wabt
+        if: matrix.name == 'wasm'
+        run: sudo apt install wabt
+
       - name: build sqlite-sync
         run: make extension ${{ matrix.make && matrix.make || ''}}
 
@@ -137,11 +144,68 @@ jobs:
         with:
           path: coverage
 
+      - name: wasm npm pack
+        if: matrix.name == 'wasm'
+        run: |
+          TAG=$(git ls-remote --tags https://github.com/sqlite/sqlite-wasm.git | \
+            awk -v ver=$(make sqlite_version) -F'/' '$NF ~ ver"-build[0-9]+$" {print $NF}' | \
+            sort -V | \
+            tail -n1)
+          git clone --branch "$TAG" --depth 1 https://github.com/sqlite/sqlite-wasm.git sqlite-wasm
+          rm -rf sqlite-wasm/sqlite-wasm/*
+          unzip dist/sqlite-wasm.zip -d sqlite-wasm/tmp
+          mv sqlite-wasm/tmp/sqlite-wasm-*/jswasm sqlite-wasm/sqlite-wasm
+
+          cd sqlite-wasm && npm i && npm run fix && npm run publint && npm run check-types && cd ..
+
+          PKG=sqlite-wasm/package.json
+          TMP=sqlite-wasm/package.tmp.json
+          
+          DESC="SQLite Wasm compiled with the automatically initialized cloudsync extension. Conveniently packaged as an ES Module for effortless integration."
+          
+          jq \
+            --arg name "@sqliteai/cloudsync-wasm" \
+            --arg version "$(make sqlite_version)-cloudsync-$(make version)" \
+            --arg desc "$DESC" \
+            --argjson keywords '["offsync","cloudsync","sqliteai"]' \
+            --arg repo_url "git+https://github.com/sqliteai/sqlite-sync.git" \
+            --arg author "Gioele Cantoni ([email protected])" \
+            --arg bugs_url "https://github.com/sqliteai/sqlite-sync/issues" \
+            --arg homepage "https://github.com/sqliteai/sqlite-sync#readme" \
+            '
+            .name = $name
+            | .version = $version
+            | .description = $desc
+            | .keywords += $keywords
+            | del(.bin)
+            | .scripts |= with_entries(select(
+                  .key != "build"
+                and .key != "start"
+                and .key != "start:node"
+                and .key != "prepublishOnly"
+                and .key != "deploy"
+              ))
+            | .repository.url = $repo_url
+            | .author = $author
+            | .bugs.url = $bugs_url
+            | .homepage = $homepage
+            | del(.devDependencies.decompress)
+            | del(.devDependencies["http-server"])
+            | del(.devDependencies.shx)
+            ' "$PKG" > "$TMP" && mv "$TMP" "$PKG"
+
+          sed 's/@sqlite\.org\/sqlite-wasm/@sqliteai\/cloudsync-wasm/g' sqlite-wasm/README.md > sqlite-wasm/README.tmp
+          echo -e "# sqlite-sync WASM $(make version)\nThis README and the TypeScript types are from the [official SQLite wasm repository](https://github.com/sqlite/sqlite-wasm)\n\n$(cat sqlite-wasm/README.tmp)" > sqlite-wasm/README.md
+          rm -rf sqlite-wasm/tmp sqlite-wasm/bin sqlite-wasm/demo sqlite-wasm/README.tmp sqlite-wasm/package-lock.json
+          cd sqlite-wasm && npm pack
+
       - uses: actions/[email protected]
         if: always()
         with:
           name: cloudsync-${{ matrix.name }}${{ matrix.arch && format('-{0}', matrix.arch) || '' }}
-          path: dist/cloudsync.*
+          path: |
+            dist/cloudsync.*
+            sqlite-wasm/*.tgz
           if-no-files-found: error
 
   release:
@@ -170,8 +234,7 @@ jobs:
       - name: release tag version from cloudsync.h
         id: tag
         run: |
-          FILE="src/cloudsync.h"
-          VERSION=$(grep -oP '#define CLOUDSYNC_VERSION\s+"\K[^"]+' "$FILE")
+          VERSION=$(make version)
           if [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
             LATEST=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" https://api.github.com/repos/${{ github.repository }}/releases/latest | jq -r '.name')
             if [[ "$VERSION" != "$LATEST" || "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]]; then
@@ -183,6 +246,18 @@ jobs:
           fi
           echo "❌ CLOUDSYNC_VERSION not found in cloudsync.h"
           exit 1
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20.x'
+          registry-url: 'https://registry.npmjs.org'
+
+      - name: wasm publish to npmjs
+        if: steps.tag.outputs.version != ''
+        #use this version when the repo will become public run: npm publish --provenance --access public ./artifacts/cloudsync-wasm/sqlite-wasm/sqliteai-cloudsync-wasm-*-cloudsync-*.tgz
+        run: npm publish --access public ./artifacts/cloudsync-wasm/sqlite-wasm/sqliteai-cloudsync-wasm-*-cloudsync-*.tgz
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
 
       - name: zip artifacts
         run: |

Makefile

@@ -8,6 +8,9 @@ SQLITE3 ?= sqlite3
 # set curl version to download and build
 CURL_VERSION ?= 8.12.1
 
+# set sqlite version for WASM static build
+SQLITE_VERSION ?= 3.49.2
+
 # Set default platform if not specified
 ifeq ($(OS),Windows_NT)
     PLATFORM := windows
@@ -47,13 +50,14 @@ CURL_DIR = curl
 CURL_SRC = $(CURL_DIR)/src/curl-$(CURL_VERSION)
 COV_DIR = coverage
 CUSTOM_CSS = $(TEST_DIR)/sqliteai.css
+BUILD_WASM = build/wasm
 
 SRC_FILES = $(wildcard $(SRC_DIR)/*.c)
 TEST_SRC = $(wildcard $(TEST_DIR)/*.c)
 TEST_FILES = $(SRC_FILES) $(TEST_SRC) $(wildcard $(SQLITE_DIR)/*.c)
 RELEASE_OBJ = $(patsubst %.c, $(BUILD_RELEASE)/%.o, $(notdir $(SRC_FILES)))
 TEST_OBJ = $(patsubst %.c, $(BUILD_TEST)/%.o, $(notdir $(TEST_FILES)))
-COV_FILES = $(filter-out $(SRC_DIR)/lz4.c $(SRC_DIR)/network.c, $(SRC_FILES))
+COV_FILES = $(filter-out $(SRC_DIR)/lz4.c $(SRC_DIR)/network.c $(SRC_DIR)/wasm.c, $(SRC_FILES))
 CURL_LIB = $(CURL_DIR)/$(PLATFORM)/libcurl.a
 TEST_TARGET = $(patsubst %.c,$(DIST_DIR)/%$(EXE), $(notdir $(TEST_SRC)))
 
@@ -110,6 +114,8 @@ else ifeq ($(PLATFORM),isim)
     T_LDFLAGS = -framework Security
     CFLAGS += -arch x86_64 -arch arm64 $(SDK)
     CURL_CONFIG = --host=arm64-apple-darwin --with-secure-transport CFLAGS="-arch x86_64 -arch arm64 -isysroot $$(xcrun --sdk iphonesimulator --show-sdk-path) -miphonesimulator-version-min=11.0"
+else ifeq ($(PLATFORM),wasm)
+    TARGET := $(DIST_DIR)/sqlite-wasm.zip
 else # linux
     TARGET := $(DIST_DIR)/cloudsync.so
     LDFLAGS += -shared -lssl -lcrypto
@@ -127,7 +133,7 @@ endif
 # Windows .def file generation
 $(DEF_FILE):
 ifeq ($(PLATFORM),windows)
-	@echo "LIBRARY js.dll" > $@
+	@echo "LIBRARY cloudsync.dll" > $@
 	@echo "EXPORTS" >> $@
 	@echo "    sqlite3_cloudsync_init" >> $@
 endif
@@ -139,12 +145,32 @@ $(shell mkdir -p $(BUILD_DIRS) $(DIST_DIR))
 extension: $(TARGET)
 all: $(TARGET) 
 
+ifneq ($(PLATFORM),wasm)
 # Loadable library
 $(TARGET): $(RELEASE_OBJ) $(DEF_FILE) $(CURL_LIB)
 	$(CC) $(RELEASE_OBJ) $(DEF_FILE) -o $@ $(LDFLAGS)
 ifeq ($(PLATFORM),windows)
     # Generate import library for Windows
-	dlltool -D $@ -d $(DEF_FILE) -l $(DIST_DIR)/js.lib
+	dlltool -D $@ -d $(DEF_FILE) -l $(DIST_DIR)/cloudsync.lib
+endif
+else
+#WASM build
+EMSDK := $(BUILD_WASM)/emsdk
+$(EMSDK):
+	git clone https://github.com/emscripten-core/emsdk.git $(EMSDK)
+	cd $(EMSDK) && ./emsdk install latest && ./emsdk activate latest
+
+SQLITE_SRC := $(BUILD_WASM)/sqlite
+$(SQLITE_SRC): $(EMSDK)
+	git clone --branch version-$(SQLITE_VERSION) --depth 1 https://github.com/sqlite/sqlite.git $(SQLITE_SRC)
+	cd $(EMSDK) && . ./emsdk_env.sh && cd ../sqlite && ./configure --enable-all
+
+WASM_FLAGS = emcc.jsflags += -sFETCH -pthread
+WASM_MAKEFILE = $(SQLITE_SRC)/ext/wasm/GNUmakefile
+$(TARGET): $(SQLITE_SRC) $(SRC_FILES)
+	@grep '$(WASM_FLAGS)' '$(WASM_MAKEFILE)' >/dev/null 2>&1 || echo '$(WASM_FLAGS)' >> '$(WASM_MAKEFILE)'
+	cd $(SQLITE_SRC)/ext/wasm && $(MAKE) dist sqlite3_wasm_extra_init.c=../../../../../src/wasm.c
+	mv $(SQLITE_SRC)/ext/wasm/sqlite-wasm-*.zip $(TARGET)
 endif
 
 # Test executable
@@ -266,9 +292,15 @@ endif
 	mv $(CURL_SRC)/lib/.libs/libcurl.a $(CURL_DIR)/$(PLATFORM)
 	rm -rf $(CURL_DIR)/src
 
+# Tools
+sqlite_version: 
+	@echo $(SQLITE_VERSION)
+version:
+	@echo $(shell sed -n 's/^#define CLOUDSYNC_VERSION[[:space:]]*"\([^"]*\)".*/\1/p' src/cloudsync.h)
+
 # Clean up generated files
 clean:
-	rm -rf $(BUILD_DIRS) $(DIST_DIR)/* $(COV_DIR) *.gcda *.gcno *.gcov $(CURL_DIR)/src *.sqlite
+	rm -rf $(BUILD_DIRS) $(DIST_DIR)/* $(COV_DIR) *.gcda *.gcno *.gcov $(CURL_DIR)/src *.sqlite $(BUILD_WASM)
 
 # Help message
 help:
@@ -283,6 +315,7 @@ help:
 	@echo "  android (needs ARCH to be set to x86_64 or arm64-v8a and ANDROID_NDK to be set)"
 	@echo "  ios (only on macOS)"
 	@echo "  isim (only on macOS)"
+	@echo "  wasm (needs wabt[brew install wabt/sudo apt install wabt])"
 	@echo ""
 	@echo "Targets:"
 	@echo "  all       				- Build the extension (default)"

src/cloudsync.c

@@ -360,7 +360,7 @@ char *db_version_build_query (sqlite3 *db) {
 
     // the good news is that the query can be computed in SQLite without the need to do any extra computation from the host language
     const char *sql = "WITH table_names AS ("
-                      "SELECT format(\"%w\", name) as tbl_name "
+                      "SELECT format('%w', name) as tbl_name "
                       "FROM sqlite_master "
                       "WHERE type='table' "
                       "AND name LIKE '%_cloudsync'"

src/cloudsync.h

@@ -16,7 +16,7 @@
 #include "sqlite3.h"
 #endif
 
-#define CLOUDSYNC_VERSION                       "0.8.5"
+#define CLOUDSYNC_VERSION                       "0.8.6"
 
 int sqlite3_cloudsync_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi);
 

src/dbutils.c

@@ -419,7 +419,7 @@ bool dbutils_table_sanity_check (sqlite3 *db, sqlite3_context *context, const ch
             // the affinity of a column is determined by the declared type of the column,
             // according to the following rules in the order shown:
             // 1. If the declared type contains the string "INT" then it is assigned INTEGER affinity.
-            sql = sqlite3_snprintf((int)blen, buffer, "SELECT count(*) FROM pragma_table_info('%w') WHERE pk=1 AND \"type\" LIKE \"%%INT%%\";", name);
+            sql = sqlite3_snprintf((int)blen, buffer, "SELECT count(*) FROM pragma_table_info('%w') WHERE pk=1 AND \"type\" LIKE '%%INT%%';", name);
             sqlite3_int64 count2 = dbutils_int_select(db, sql);
             if (count == count2) {
                 dbutils_context_result_error(context, "Table %s uses an single-column INTEGER primary key. For CRDT replication, primary keys must be globally unique. Consider using a TEXT primary key with UUIDs or ULID to avoid conflicts across nodes. If you understand the risk and still want to use this INTEGER primary key, set the third argument of the cloudsync_init function to 1 to skip this check.", name);