perf: speed up query for filling missing columns in refill metatable (#18)

The new query does 1 encode per source row and one indexed NOT-EXISTS probe.
The old plan does many decodes per candidate and can’t use an index to rule out matches quickly—so it burns CPU and I/O.

* Increased performance in cloudsync_refill_metatable
* chore: remove old code
* chore: bump version

---------

Co-authored-by: Marco Bambini <[email protected]>

Author: andinux

src/cloudsync.c

@@ -1711,25 +1711,22 @@ int cloudsync_refill_metatable (sqlite3 *db, cloudsync_context *data, const char
     char *pkdecode = dbutils_text_select(db, sql);
     char *pkdecodeval = (pkdecode) ? pkdecode : "cloudsync_pk_decode(pk, 1) AS rowid";
     cloudsync_memory_free(sql);
-    
-    sql = cloudsync_memory_mprintf("SELECT group_concat('\"' || format('%%w', name) || '\"' || ' = cloudsync_pk_decode(pk, ' || pk || ')', ' AND ') FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk;", table_name);
-    char *pkonclause = dbutils_text_select(db, sql);
-    char *pkonclauseval = (pkonclause) ? pkonclause : "rowid = cloudsync_pk_decode(pk, 1) AS rowid";
-    cloudsync_memory_free(sql);
-    
+     
     sql = cloudsync_memory_mprintf("SELECT cloudsync_insert('%q', %s) FROM (SELECT %s FROM \"%w\" EXCEPT SELECT %s FROM \"%w_cloudsync\");", table_name, pkvalues_identifiers, pkvalues_identifiers, table_name, pkdecodeval, table_name);
     int rc = sqlite3_exec(db, sql, NULL, NULL, NULL);
     cloudsync_memory_free(sql);
     if (rc != SQLITE_OK) goto finalize;
-
+    
     // fill missing colums
     // for each non-pk column:
+    // The new query does 1 encode per source row and one indexed NOT-EXISTS probe.
+    // The old plan does many decodes per candidate and can’t use an index to rule out matches quickly—so it burns CPU and I/O.
 
-    sql = cloudsync_memory_mprintf("SELECT cloudsync_pk_encode(%s) FROM \"%w\" LEFT JOIN \"%w_cloudsync\" ON %s AND \"%w_cloudsync\".col_name = ? WHERE \"%w_cloudsync\".db_version IS NULL", pkvalues_identifiers, table_name, table_name, pkonclauseval, table_name, table_name);
-    rc = sqlite3_prepare(db, sql, -1, &vm, NULL);
+    sql = cloudsync_memory_mprintf("WITH _cstemp1 AS (SELECT cloudsync_pk_encode(%s) AS pk FROM \"%w\") SELECT _cstemp1.pk FROM _cstemp1 WHERE NOT EXISTS (SELECT 1 FROM \"%w_cloudsync\" _cstemp2 WHERE _cstemp2.pk = _cstemp1.pk AND _cstemp2.col_name = ?);", pkvalues_identifiers, table_name, table_name);
+    rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &vm, NULL);
     cloudsync_memory_free(sql);
     if (rc != SQLITE_OK) goto finalize;
-    
+     
     for (int i=0; i<table->ncols; ++i) {
         char *col_name = table->col_name[i];
 
@@ -1758,7 +1755,6 @@ int cloudsync_refill_metatable (sqlite3 *db, cloudsync_context *data, const char
     if (rc != SQLITE_OK) DEBUG_ALWAYS("cloudsync_refill_metatable error: %s", sqlite3_errmsg(db));
     if (pkclause_identifiers) cloudsync_memory_free(pkclause_identifiers);
     if (pkdecode) cloudsync_memory_free(pkdecode);
-    if (pkonclause) cloudsync_memory_free(pkonclause);
     if (vm) sqlite3_finalize(vm);
     return rc;
 }

src/cloudsync.h

@@ -20,7 +20,7 @@
 extern "C" {
 #endif
 
-#define CLOUDSYNC_VERSION                       "0.8.41"
+#define CLOUDSYNC_VERSION                       "0.8.51"
 
 int sqlite3_cloudsync_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi);
 

test/unit.c

@@ -5930,7 +5930,7 @@ int main(int argc, const char * argv[]) {
     cloudsync_set_payload_apply_callback(db, unittest_payload_apply_rls_callback);
 
     printf("Testing CloudSync version %s\n", CLOUDSYNC_VERSION);
-    printf("===============================\n");
+    printf("=================================\n");
 
     result += test_report("PK Test:", do_test_pk(db, 10000, print_result));
     result += test_report("UUID Test:", do_test_uuid(db, 1000, print_result));