new integration test (#4)

* feat: init integration tests

* fix: optimize build commands and improve CPU utilization in Makefile

* New Android PEM specific code

* fix android cert: add new line

* refactor: remove health-track schema and update db_init function for improved initialization

* refactor: improve error handling and add in memory peers test

* refactor: multi-threaded sync testing with random text insert

* expect sync to be gt0 only when not initializing the db

* to fix test

* test only 5 peers

* test 2 peers

* test: expect changes gt0 only when not initializing the db

* test: low peers count

* fix integration test: add windows thread handling

* fix: rename INT to INTGR in expected_type enum to avoid redeclaration issues

* test 1 peer

* add max retry and wait time

* increase wait time and retries

* Resolve cacert.h merge conflicts

* test print network sync results

* sync test skip insert

* test print network init string

* test: remove debug prints

* test: update PEERS count and modify cloudsync_network_sync parameters

* test: lower peers count

* test: missing close database connection after initialization

* linux error test

* revert last test commit

* test debug functions and settings

* test 1 peer

* test 2 peer

* print cloudsync_network_sync

* test: disable debug flags

* test more peers

* final test 50 peers

* remove debug network sync print

* test 10 peers

* expect network sync to be gt0

* hardcode test executables

* fix makefile link pthread

* revert last commits

* lower peers count

---------

Co-authored-by: Marco Bambini <[email protected]>

Author: Gioee

.github/workflows/main.yml

@@ -48,6 +48,11 @@ jobs:
       run:
         shell: bash
 
+    env:
+      CONNECTION_STRING: ${{ secrets.CONNECTION_STRING }}
+      APIKEY: ${{ secrets.APIKEY }}
+      WEBLITE: ${{ secrets.WEBLITE }}
+
     steps:
 
       - uses: actions/[email protected]
@@ -62,11 +67,11 @@ jobs:
 
       - name: windows build curl
         if: matrix.os == 'windows-latest'
-        run: make curl/windows/libcurl.a -j4
+        run: make curl/windows/libcurl.a
         shell: msys2 {0}
 
       - name: build sqlite-sync
-        run: make extension ${{ matrix.make && matrix.make || ''}} -j4
+        run: make extension ${{ matrix.make && matrix.make || ''}}
 
       - name: windows install sqlite3
         if: matrix.os == 'windows-latest'
@@ -96,10 +101,13 @@ jobs:
           echo "::endgroup::"
 
           echo "::group::prepare the test script"
-          make test PLATFORM=$PLATFORM ARCH=$ARCH -j4 || echo "It should fail. Running remaining commands in the emulator"
+          make test PLATFORM=$PLATFORM ARCH=$ARCH || echo "It should fail. Running remaining commands in the emulator"
           cat > commands.sh << EOF
             mv -f /data/local/tmp/sqlite3 /system/xbin
             cd /data/local/tmp
+            export CONNECTION_STRING="$CONNECTION_STRING"
+            export APIKEY="$APIKEY"
+            export WEBLITE="$WEBLITE"
             $(make test PLATFORM=$PLATFORM ARCH=$ARCH -n)
           EOF
           echo "::endgroup::"
@@ -118,7 +126,7 @@ jobs:
 
       - name: test sqlite-sync
         if: matrix.name == 'linux' || matrix.name == 'windows'
-        run: make test -j4
+        run: make test
 
       - name: test sqlite-sync + coverage
         if: matrix.name == 'macos'

.gitignore

@@ -10,3 +10,4 @@
 *.a
 unittest
 /curl/src
+.vscode

Makefile

@@ -11,16 +11,22 @@ CURL_VERSION ?= 8.12.1
 # Set default platform if not specified
 ifeq ($(OS),Windows_NT)
     PLATFORM := windows
-    HOST:= windows
+    HOST := windows
+    CPUS := $(shell powershell -Command "[Environment]::ProcessorCount")
 else
     HOST = $(shell uname -s | tr '[:upper:]' '[:lower:]')
     ifeq ($(HOST),darwin)
         PLATFORM := macos
+        CPUS := $(shell sysctl -n hw.ncpu)
     else
         PLATFORM := $(HOST)
+        CPUS := $(shell nproc)
     endif
 endif
 
+# Speed up builds by using all available CPU cores
+MAKEFLAGS += -j$(CPUS)
+
 # Compiler and flags
 CC = gcc
 CFLAGS = -Wall -Wextra -Wno-unused-parameter -I$(SRC_DIR) -I$(SQLITE_DIR) -I$(CURL_DIR)/include
@@ -42,18 +48,14 @@ CURL_SRC = $(CURL_DIR)/src/curl-$(CURL_VERSION)
 COV_DIR = coverage
 CUSTOM_CSS = $(TEST_DIR)/sqliteai.css
 
-# Files and objects
-ifeq ($(PLATFORM),windows)
-    TEST_TARGET := $(DIST_DIR)/test.exe
-else
-    TEST_TARGET := $(DIST_DIR)/test
-endif
 SRC_FILES = $(wildcard $(SRC_DIR)/*.c)
-TEST_FILES = $(SRC_FILES) $(wildcard $(TEST_DIR)/*.c) $(wildcard $(SQLITE_DIR)/*.c)
+TEST_SRC = $(wildcard $(TEST_DIR)/*.c)
+TEST_FILES = $(SRC_FILES) $(TEST_SRC) $(wildcard $(SQLITE_DIR)/*.c)
 RELEASE_OBJ = $(patsubst %.c, $(BUILD_RELEASE)/%.o, $(notdir $(SRC_FILES)))
 TEST_OBJ = $(patsubst %.c, $(BUILD_TEST)/%.o, $(notdir $(TEST_FILES)))
 COV_FILES = $(filter-out $(SRC_DIR)/lz4.c $(SRC_DIR)/network.c, $(SRC_FILES))
 CURL_LIB = $(CURL_DIR)/$(PLATFORM)/libcurl.a
+TEST_TARGET = $(patsubst %.c,$(DIST_DIR)/%$(EXE), $(notdir $(TEST_SRC)))
 
 # Platform-specific settings
 ifeq ($(PLATFORM),windows)
@@ -64,6 +66,7 @@ ifeq ($(PLATFORM),windows)
     DEF_FILE := $(BUILD_RELEASE)/cloudsync.def
     CFLAGS += -DCURL_STATICLIB
     CURL_CONFIG = --with-schannel CFLAGS="-DCURL_STATICLIB"
+    EXE = .exe
 else ifeq ($(PLATFORM),macos)
     TARGET := $(DIST_DIR)/cloudsync.dylib
     LDFLAGS += -arch x86_64 -arch arm64 -framework Security -dynamiclib -undefined dynamic_lookup
@@ -90,7 +93,7 @@ else ifeq ($(PLATFORM),android)
 
     OPENSSL := $(BIN)/../sysroot/usr/include/openssl
     CC = $(BIN)/$(ARCH)-linux-android26-clang
-    CURL_CONFIG = --host $(ARCH)-$(HOST)-android26 --with-openssl=$(BIN)/../sysroot/usr LIBS="-lssl -lcrypto" AR=$(BIN)/llvm-ar AS=$(BIN)/llvm-as CC=$(BIN)/$(ARCH)-linux-android26-clang CXX=$(BIN)/$(ARCH)-linux-android26-clang++ LD=$(BIN)/ld RANLIB=$(BIN)/llvm-ranlib STRIP=$(BIN)/llvm-strip
+    CURL_CONFIG = --host $(ARCH)-$(HOST)-android26 --with-openssl=$(BIN)/../sysroot/usr LIBS="-lssl -lcrypto" AR=$(BIN)/llvm-ar AS=$(BIN)/llvm-as CC=$(CC) CXX=$(BIN)/$(ARCH)-linux-android26-clang++ LD=$(BIN)/ld RANLIB=$(BIN)/llvm-ranlib STRIP=$(BIN)/llvm-strip
     TARGET := $(DIST_DIR)/cloudsync.so
     LDFLAGS += -shared -lcrypto -lssl
 else ifeq ($(PLATFORM),ios)
@@ -146,20 +149,20 @@ endif
 
 # Test executable
 $(TEST_TARGET): $(TEST_OBJ)
-	$(CC) $(TEST_OBJ) -o $@ $(T_LDFLAGS)
+	$(CC) $(filter-out $(patsubst $(DIST_DIR)/%$(EXE),$(BUILD_TEST)/%.o, $(filter-out $@,$(TEST_TARGET))), $(TEST_OBJ)) -o $@ $(T_LDFLAGS)
 
 # Object files
 $(BUILD_RELEASE)/%.o: %.c
 	$(CC) $(CFLAGS) -O3 -fPIC -c $< -o $@
 $(BUILD_TEST)/sqlite3.o: $(SQLITE_DIR)/sqlite3.c
-	$(CC) $(CFLAGS) -DSQLITE_CORE=1 -c $< -o $@
+	$(CC) $(CFLAGS) -DSQLITE_CORE -c $< -o $@
 $(BUILD_TEST)/%.o: %.c
 	$(CC) $(T_CFLAGS) -c $< -o $@
 
 # Run code coverage (--css-file $(CUSTOM_CSS))
 test: $(TARGET) $(TEST_TARGET)
 	$(SQLITE3) ":memory:" -cmd ".bail on" ".load ./$<" "SELECT cloudsync_version();"
-	./$(TEST_TARGET)
+	set -e; for t in $(TEST_TARGET); do ./$$t; done
 ifneq ($(COVERAGE),false)
 	mkdir -p $(COV_DIR)
 	lcov --capture --directory . --output-file $(COV_DIR)/coverage.info $(subst src, --include src,${COV_FILES})
@@ -265,7 +268,7 @@ endif
 
 # Clean up generated files
 clean:
-	rm -rf $(BUILD_DIRS) $(DIST_DIR)/* $(COV_DIR) *.gcda *.gcno *.gcov $(CURL_DIR)/src
+	rm -rf $(BUILD_DIRS) $(DIST_DIR)/* $(COV_DIR) *.gcda *.gcno *.gcov $(CURL_DIR)/src *.sqlite
 
 # Help message
 help: