Added a way to load/save payload from file (available only on Desktop OSes)

Author: marcobambini

src/cloudsync.c

@@ -2258,6 +2258,108 @@ void cloudsync_payload_decode (sqlite3_context *context, int argc, sqlite3_value
     cloudsync_payload_apply(context, payload, blen);
 }
 
+// MARK: - Payload load/store -
+
+#ifdef CLOUDSYNC_DESKTOP_OS
+
+int cloudsync_payload_get (sqlite3_context *context, char **blob, int *blob_size, int *db_version, int *seq, sqlite3_int64 *new_db_version, sqlite3_int64 *new_seq) {
+    sqlite3 *db = sqlite3_context_db_handle(context);
+
+    *db_version = dbutils_settings_get_int_value(db, CLOUDSYNC_KEY_SEND_DBVERSION);
+    if (*db_version < 0) {sqlite3_result_error(context, "Unable to retrieve db_version.", -1); return SQLITE_ERROR;}
+
+    *seq = dbutils_settings_get_int_value(db, CLOUDSYNC_KEY_SEND_SEQ);
+    if (*seq < 0) {sqlite3_result_error(context, "Unable to retrieve seq.", -1); return SQLITE_ERROR;}
+    
+    // retrieve BLOB
+    char sql[1024];
+    snprintf(sql, sizeof(sql), "WITH max_db_version AS (SELECT MAX(db_version) AS max_db_version FROM cloudsync_changes) "
+                               "SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), max_db_version AS max_db_version, MAX(IIF(db_version = max_db_version, seq, NULL)) FROM cloudsync_changes, max_db_version WHERE site_id=cloudsync_siteid() AND (db_version>%d OR (db_version=%d AND seq>%d))", *db_version, *db_version, *seq);
+    
+    int rc = dbutils_blob_int_int_select(db, sql, blob, blob_size, new_db_version, new_seq);
+    if (rc != SQLITE_OK) {
+        sqlite3_result_error(context, "cloudsync_network_send_changes unable to get changes", -1);
+        sqlite3_result_error_code(context, rc);
+        return rc;
+    }
+    
+    // exit if there is no data to send
+    if (blob == NULL || blob_size == 0) return SQLITE_OK;
+    return rc;
+}
+
+void cloudsync_payload_save (sqlite3_context *context, int argc, sqlite3_value **argv) {
+    DEBUG_FUNCTION("cloudsync_payload_save");
+    
+    // sanity check argument
+    if (sqlite3_value_type(argv[0]) != SQLITE_TEXT) {
+        sqlite3_result_error(context, "Unable to retrieve file path.", -1);
+        return;
+    }
+    
+    // retrieve full path to file
+    const char *path = (const char *)sqlite3_value_text(argv[0]);
+    file_delete(path);
+    
+    // retrieve payload
+    char *blob = NULL;
+    int blob_size = 0, db_version = 0, seq = 0;
+    sqlite3_int64 new_db_version = 0, new_seq = 0;
+    int rc = cloudsync_payload_get(context, &blob, &blob_size, &db_version, &seq, &new_db_version, &new_seq);
+    if (rc != SQLITE_OK) return;
+    
+    // exit if there is no data to send
+    if (blob == NULL || blob_size == 0) return;
+    
+    // write payload to file
+    bool res = file_write(path, blob, (size_t)blob_size);
+    sqlite3_free(blob);
+    
+    if (res == false) {
+        sqlite3_result_error(context, "Unable to write payload to file path.", -1);
+        return;
+    }
+    
+    // update db_version and seq
+    char buf[256];
+    sqlite3 *db = sqlite3_context_db_handle(context);
+    if (new_db_version != db_version) {
+        snprintf(buf, sizeof(buf), "%lld", new_db_version);
+        dbutils_settings_set_key_value(db, context, CLOUDSYNC_KEY_SEND_DBVERSION, buf);
+    }
+    if (new_seq != seq) {
+        snprintf(buf, sizeof(buf), "%lld", new_seq);
+        dbutils_settings_set_key_value(db, context, CLOUDSYNC_KEY_SEND_SEQ, buf);
+    }
+}
+
+void cloudsync_payload_load (sqlite3_context *context, int argc, sqlite3_value **argv) {
+    DEBUG_FUNCTION("cloudsync_payload_load");
+    
+    // sanity check argument
+    if (sqlite3_value_type(argv[0]) != SQLITE_TEXT) {
+        sqlite3_result_error(context, "Unable to retrieve file path.", -1);
+        return;
+    }
+    
+    // retrieve full path to file
+    const char *path = (const char *)sqlite3_value_text(argv[0]);
+    file_delete(path);
+    
+    size_t payload_size = 0;
+    char *payload = file_read(path, &payload_size);
+    if (!payload) {
+        if (payload_size == -1) sqlite3_result_error(context, "Unable to read payload from file path.", -1);
+        return;
+    }
+    
+    int nrows = cloudsync_payload_apply (context, payload, (int)payload_size);
+    if (payload) cloudsync_memory_free(payload);
+    if (nrows != -1) sqlite3_result_int(context, nrows);
+}
+
+#endif
+
 // MARK: - Public -
 
 void cloudsync_version (sqlite3_context *context, int argc, sqlite3_value **argv) {
@@ -3331,6 +3433,14 @@ int cloudsync_register (sqlite3 *db, char **pzErrMsg) {
     rc = dbutils_register_function(db, "cloudsync_payload_decode", cloudsync_payload_decode, -1, pzErrMsg, ctx, NULL);
     if (rc != SQLITE_OK) return rc;
 
+    #ifdef CLOUDSYNC_DESKTOP_OS
+    rc = dbutils_register_function(db, "cloudsync_payload_save", cloudsync_payload_save, 1, pzErrMsg, ctx, NULL);
+    if (rc != SQLITE_OK) return rc;
+    
+    rc = dbutils_register_function(db, "cloudsync_payload_load", cloudsync_payload_load, 1, pzErrMsg, ctx, NULL);
+    if (rc != SQLITE_OK) return rc;
+    #endif
+    
     // PRIVATE functions
     rc = dbutils_register_function(db, "cloudsync_is_sync", cloudsync_is_sync, 1, pzErrMsg, ctx, NULL);
     if (rc != SQLITE_OK) return rc;

src/cloudsync.h

@@ -20,7 +20,7 @@
 extern "C" {
 #endif
 
-#define CLOUDSYNC_VERSION                       "0.8.32"
+#define CLOUDSYNC_VERSION                       "0.8.33"
 
 int sqlite3_cloudsync_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi);
 

src/cloudsync_private.h

@@ -37,6 +37,7 @@ const char *cloudsync_context_init (sqlite3 *db, cloudsync_context *data, sqlite
 void *cloudsync_get_auxdata (sqlite3_context *context);
 void cloudsync_set_auxdata (sqlite3_context *context, void *xdata);
 int cloudsync_payload_apply (sqlite3_context *context, const char *payload, int blen);
+int cloudsync_payload_get (sqlite3_context *context, char **blob, int *blob_size, int *db_version, int *seq, sqlite3_int64 *new_db_version, sqlite3_int64 *new_seq);
 
 // used by core
 typedef bool (*cloudsync_payload_apply_callback_t)(void **xdata, cloudsync_pk_decode_bind_context *decoded_change, sqlite3 *db, cloudsync_context *data, int step, int rc);

src/network.c

@@ -693,28 +693,12 @@ int cloudsync_network_send_changes_internal (sqlite3_context *context, int argc,
     network_data *data = (network_data *)cloudsync_get_auxdata(context);
     if (!data) {sqlite3_result_error(context, "Unable to retrieve CloudSync context.", -1); return SQLITE_ERROR;}
 
-    sqlite3 *db = sqlite3_context_db_handle(context);
-
-    int db_version = dbutils_settings_get_int_value(db, CLOUDSYNC_KEY_SEND_DBVERSION);
-    if (db_version<0) {sqlite3_result_error(context, "Unable to retrieve db_version.", -1); return SQLITE_ERROR;}
-
-    int seq = dbutils_settings_get_int_value(db, CLOUDSYNC_KEY_SEND_SEQ);
-    if (seq<0) {sqlite3_result_error(context, "Unable to retrieve seq.", -1); return SQLITE_ERROR;}
-    
-    // retrieve BLOB
-    char sql[1024];
-    snprintf(sql, sizeof(sql), "WITH max_db_version AS (SELECT MAX(db_version) AS max_db_version FROM cloudsync_changes) "
-                               "SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), max_db_version AS max_db_version, MAX(IIF(db_version = max_db_version, seq, NULL)) FROM cloudsync_changes, max_db_version WHERE site_id=cloudsync_siteid() AND (db_version>%d OR (db_version=%d AND seq>%d))", db_version, db_version, seq);
-    int blob_size = 0;
+    // retrieve payload
     char *blob = NULL;
-    sqlite3_int64 new_db_version = 0;
-    sqlite3_int64 new_seq = 0;
-    int rc = dbutils_blob_int_int_select(db, sql, &blob, &blob_size, &new_db_version, &new_seq);
-    if (rc != SQLITE_OK) {
-        sqlite3_result_error(context, "cloudsync_network_send_changes unable to get changes", -1);
-        sqlite3_result_error_code(context, rc);
-        return rc;
-    }
+    int blob_size = 0, db_version = 0, seq = 0;
+    sqlite3_int64 new_db_version = 0, new_seq = 0;
+    int rc = cloudsync_payload_get(context, &blob, &blob_size, &db_version, &seq, &new_db_version, &new_seq);
+    if (rc != SQLITE_OK) return rc;
 
     // exit if there is no data to send
     if (blob == NULL || blob_size == 0) return SQLITE_OK;
@@ -749,7 +733,9 @@ int cloudsync_network_send_changes_internal (sqlite3_context *context, int argc,
         return SQLITE_ERROR;
     }
 
+    // update db_version and seq
     char buf[256];
+    sqlite3 *db = sqlite3_context_db_handle(context);
     if (new_db_version != db_version) {
         snprintf(buf, sizeof(buf), "%lld", new_db_version);
         dbutils_settings_set_key_value(db, context, CLOUDSYNC_KEY_SEND_DBVERSION, buf);

src/utils.c

@@ -14,13 +14,23 @@
 #include <objbase.h>
 #include <bcrypt.h>
 #include <ntstatus.h> //for STATUS_SUCCESS
+#include <io.h>
+#define file_close      _close
 #else
 #include <unistd.h>
 #if defined(__APPLE__)
 #include <Security/Security.h>
 #elif !defined(__ANDROID__)
 #include <sys/random.h>
 #endif
+#define file_close      close
+#endif
+
+#ifdef CLOUDSYNC_DESKTOP_OS
+#include <fcntl.h>
+#include <errno.h>
+#include <sys/stat.h>
+#include <sys/types.h>
 #endif
 
 #ifndef SQLITE_CORE
@@ -280,6 +290,124 @@ uint64_t fnv1a_hash (const char *data, size_t len) {
 
     return h_final;
 }
+
+// MARK: - Files -
+
+#ifdef CLOUDSYNC_DESKTOP_OS
+
+bool file_delete (const char *path) {
+    #ifdef _WIN32
+    return DeleteFileA(path);
+    #else
+    return (unlink(path) == 0);
+    #endif
+}
+
+static bool file_read_all (int fd, char *buf, size_t n) {
+    size_t off = 0;
+    while (off < n) {
+        #ifdef _WIN32
+        int r = _read(fd, buf + off, (unsigned)(n - off));
+        if (r <= 0) return false;
+        #else
+        ssize_t r = read(fd, buf + off, n - off);
+        if (r < 0) {
+            if (errno == EINTR) continue;
+            return false;
+        }
+        if (r == 0) return false; // unexpected EOF
+        #endif
+        off += (size_t)r;
+    }
+    return true;
+}
+
+char *file_read (const char *path, size_t *len) {
+    int fd = -1;
+    char *buffer = NULL;
+
+    #ifdef _WIN32
+    fd = _open(path, _O_RDONLY | _O_BINARY);
+    #else
+    fd = open(path, O_RDONLY);
+    #endif
+    if (fd < 0) goto abort_read;
+
+    // Get size after open to reduce TOCTTOU
+    #ifdef _WIN32
+    struct _stat64 st;
+    if (_fstat64(fd, &st) != 0 || st.st_size < 0) goto abort_read;
+    int64_t isz = st.st_size;
+    #else
+    struct stat st;
+    if (fstat(fd, &st) != 0 || st.st_size < 0) goto abort_read;
+    int64_t isz = st.st_size;
+    #endif
+
+    size_t sz = (size_t)isz;
+    // optional: guard against huge files that don't fit in size_t
+    if ((int64_t)sz != isz) goto abort_read;
+
+    buffer = (char *)cloudsync_memory_alloc(sz + 1);
+    if (!buffer) goto abort_read;
+    buffer[sz] = '\0';
+
+    if (!file_read_all(fd, buffer, sz)) goto abort_read;
+    if (len) *len = sz;
+    
+    file_close(fd);
+    return buffer;
+
+abort_read:
+    if (len) *len = -1;
+    if (buffer) cloudsync_memory_free(buffer);
+    if (fd >= 0) file_close(fd);
+    return NULL;
+}
+
+int file_create (const char *path) {
+    #ifdef _WIN32
+    int flags = _O_WRONLY | _O_CREAT | _O_TRUNC | _O_BINARY;
+    int mode  = _S_IWRITE; // Windows ignores most POSIX perms
+    return _open(path, flags, mode);
+    #else
+    int flags = O_WRONLY | O_CREAT | O_TRUNC;
+    mode_t mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH;
+    return open(path, flags, mode);
+    #endif
+}
+
+static bool file_write_all (int fd, const char *buf, size_t n) {
+    size_t off = 0;
+    while (off < n) {
+        #ifdef _WIN32
+        int w = _write(fd, buf + off, (unsigned)(n - off));
+        if (w <= 0) return false;
+        #else
+        ssize_t w = write(fd, buf + off, n - off);
+        if (w < 0) {
+            if (errno == EINTR) continue;
+            return false;
+        }
+        if (w == 0) return false;
+        #endif
+        off += (size_t)w;
+    }
+    return true;
+}
+
+bool file_write (const char *path, const char *buffer, size_t len) {
+    int fd = file_create(path);
+    if (fd < 0) return false;
+    
+    bool res = file_write_all(fd, buffer, len);
+    
+    file_close(fd);
+    return res;
+}
+
+#endif
+
 // MARK: - CRDT algos -
 
 table_algo crdt_algo_from_name (const char *algo_name) {

src/utils.h

@@ -15,6 +15,19 @@
 #include <strings.h>
 #include <string.h>
 
+// CLOUDSYNC_DESKTOP_OS = 1 if compiling for macOS, Linux (desktop), or Windows
+// Not set for iOS, Android, WebAssembly, or other platforms
+#if defined(_WIN32) && !defined(__ANDROID__) && !defined(__EMSCRIPTEN__)
+    #define CLOUDSYNC_DESKTOP_OS 1
+#elif defined(__APPLE__) && defined(__MACH__)
+    #include <TargetConditionals.h>
+    #if TARGET_OS_OSX
+        #define CLOUDSYNC_DESKTOP_OS 1
+    #endif
+#elif defined(__linux__) && !defined(__ANDROID__) && !defined(__EMSCRIPTEN__)
+    #define CLOUDSYNC_DESKTOP_OS 1
+#endif
+
 #ifndef SQLITE_CORE
 #include "sqlite3ext.h"
 #else
@@ -135,4 +148,11 @@ int cloudsync_blob_compare(const char *blob1, size_t size1, const char *blob2, s
 
 void cloudsync_rowid_decode (sqlite3_int64 rowid, sqlite3_int64 *db_version, sqlite3_int64 *seq);
 
+// available only on Desktop OS
+#ifdef CLOUDSYNC_DESKTOP_OS
+bool file_delete (const char *path);
+char *file_read (const char *path, size_t *len);
+bool file_write (const char *path, const char *buffer, size_t len);
+#endif
+
 #endif