Skip to content

Commit c278bdc

Browse files
andinuxandinux
authored
Merge pull request #15 from sqliteai/feature/unlimited-columns
Added support for tables with more than 64 columns (previously limited to 64).
2 parents 74553d3 + 1db3ee2 commit c278bdc

File tree

4 files changed

+328
-79
lines changed

4 files changed

+328
-79
lines changed

‎src/cloudsync.c‎

Lines changed: 130 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -226,6 +226,14 @@ typedef struct PACKED {
226226
uint8_t unused[6]; // padding to ensure the struct is exactly 32 bytes
227227
} cloudsync_network_header;
228228

229+
typedef struct {
230+
sqlite3_value *table_name;
231+
sqlite3_value **new_values;
232+
sqlite3_value **old_values;
233+
int count;
234+
int capacity;
235+
} cloudsync_update_payload;
236+
229237
#ifdef _MSC_VER
230238
#pragma pack(pop)
231239
#endif
@@ -2545,16 +2553,10 @@ void cloudsync_insert (sqlite3_context *context, int argc, sqlite3_value **argv)
25452553
if (pk != buffer) cloudsync_memory_free(pk);
25462554
}
25472555

2548-
void cloudsync_update (sqlite3_context *context, int argc, sqlite3_value **argv) {
2549-
DEBUG_FUNCTION("cloudsync_update %s", sqlite3_value_text(argv[0]));
2556+
void cloudsync_delete (sqlite3_context *context, int argc, sqlite3_value **argv) {
2557+
DEBUG_FUNCTION("cloudsync_delete %s", sqlite3_value_text(argv[0]));
25502558
// debug_values(argc-1, &argv[1]);
25512559

2552-
// arguments are:
2553-
// [0] table name
2554-
// [1..table->npks] NEW.prikeys
2555-
// [1+table->npks ..] OLD.prikeys
2556-
// then NEW.value,OLD.value
2557-
25582560
// retrieve context
25592561
sqlite3 *db = sqlite3_context_db_handle(context);
25602562
cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context);
@@ -2563,31 +2565,139 @@ void cloudsync_update (sqlite3_context *context, int argc, sqlite3_value **argv)
25632565
const char *table_name = (const char *)sqlite3_value_text(argv[0]);
25642566
cloudsync_table_context *table = table_lookup(data, table_name);
25652567
if (!table) {
2566-
dbutils_context_result_error(context, "Unable to retrieve table name %s in cloudsync_update.", table_name);
2568+
dbutils_context_result_error(context, "Unable to retrieve table name %s in cloudsync_delete.", table_name);
25672569
return;
25682570
}
25692571

25702572
// compute the next database version for tracking changes
25712573
sqlite3_int64 db_version = db_version_next(db, data, CLOUDSYNC_VALUE_NOTSET);
25722574
int rc = SQLITE_OK;
25732575

2574-
// check if the primary key(s) have changed by comparing the NEW and OLD primary key values
2576+
// encode the primary key values into a buffer
2577+
char buffer[1024];
2578+
size_t pklen = sizeof(buffer);
2579+
char *pk = pk_encode_prikey(&argv[1], table->npks, buffer, &pklen);
2580+
if (!pk) {
2581+
sqlite3_result_error(context, "Not enough memory to encode the primary key(s).", -1);
2582+
return;
2583+
}
2584+
2585+
// mark the row as deleted by inserting a delete sentinel into the metadata
2586+
rc = local_mark_delete_meta(db, table, pk, pklen, db_version, BUMP_SEQ(data));
2587+
if (rc != SQLITE_OK) goto cleanup;
2588+
2589+
// remove any metadata related to the old rows associated with this primary key
2590+
rc = local_drop_meta(db, table, pk, pklen);
2591+
if (rc != SQLITE_OK) goto cleanup;
2592+
2593+
cleanup:
2594+
if (rc != SQLITE_OK) sqlite3_result_error(context, sqlite3_errmsg(db), -1);
2595+
// free memory if the primary key was dynamically allocated
2596+
if (pk != buffer) cloudsync_memory_free(pk);
2597+
}
2598+
2599+
// MARK: -
2600+
2601+
void cloudsync_update_payload_free (cloudsync_update_payload *payload) {
2602+
for (int i=0; i<payload->count; i++) {
2603+
sqlite3_value_free(payload->new_values[i]);
2604+
sqlite3_value_free(payload->old_values[i]);
2605+
}
2606+
cloudsync_memory_free(payload->new_values);
2607+
cloudsync_memory_free(payload->old_values);
2608+
sqlite3_value_free(payload->table_name);
2609+
payload->new_values = NULL;
2610+
payload->old_values = NULL;
2611+
payload->table_name = NULL;
2612+
payload->count = 0;
2613+
payload->capacity = 0;
2614+
}
2615+
2616+
int cloudsync_update_payload_append (cloudsync_update_payload *payload, sqlite3_value *v1, sqlite3_value *v2, sqlite3_value *v3) {
2617+
if (payload->count >= payload->capacity) {
2618+
int newcap = payload->capacity ? payload->capacity * 2 : 128;
2619+
2620+
sqlite3_value **new_values_2 = (sqlite3_value **)cloudsync_memory_realloc(payload->new_values, newcap * sizeof(*new_values_2));
2621+
if (!new_values_2) return SQLITE_NOMEM;
2622+
payload->new_values = new_values_2;
2623+
2624+
sqlite3_value **old_values_2 = (sqlite3_value **)cloudsync_memory_realloc(payload->old_values, newcap * sizeof(*old_values_2));
2625+
if (!old_values_2) return SQLITE_NOMEM;
2626+
payload->old_values = old_values_2;
2627+
2628+
payload->capacity = newcap;
2629+
}
2630+
2631+
int index = payload->count;
2632+
if (payload->table_name == NULL) payload->table_name = sqlite3_value_dup(v1);
2633+
else if (dbutils_value_compare(payload->table_name, v1) != 0) return SQLITE_NOMEM;
2634+
payload->new_values[index] = sqlite3_value_dup(v2);
2635+
payload->old_values[index] = sqlite3_value_dup(v3);
2636+
payload->count++;
2637+
2638+
// sanity check memory allocations
2639+
bool v1_can_be_null = (sqlite3_value_type(v1) == SQLITE_NULL);
2640+
bool v2_can_be_null = (sqlite3_value_type(v2) == SQLITE_NULL);
2641+
bool v3_can_be_null = (sqlite3_value_type(v3) == SQLITE_NULL);
2642+
2643+
if ((payload->table_name == NULL) && (!v1_can_be_null)) return SQLITE_NOMEM;
2644+
if ((payload->old_values[index] == NULL) && (!v2_can_be_null)) return SQLITE_NOMEM;
2645+
if ((payload->new_values[index] == NULL) && (!v3_can_be_null)) return SQLITE_NOMEM;
2646+
2647+
return SQLITE_OK;
2648+
}
2649+
2650+
void cloudsync_update_step (sqlite3_context *context, int argc, sqlite3_value **argv) {
2651+
// argv[0] => table_name
2652+
// argv[1] => new_column_value
2653+
// argv[2] => old_column_value
2654+
2655+
// allocate/get the update payload
2656+
cloudsync_update_payload *payload = (cloudsync_update_payload *)sqlite3_aggregate_context(context, sizeof(cloudsync_update_payload));
2657+
if (!payload) {sqlite3_result_error_nomem(context); return;}
2658+
2659+
if (cloudsync_update_payload_append(payload, argv[0], argv[1], argv[2]) != SQLITE_OK) {
2660+
sqlite3_result_error_nomem(context);
2661+
}
2662+
}
2663+
2664+
void cloudsync_update_final (sqlite3_context *context) {
2665+
cloudsync_update_payload *payload = (cloudsync_update_payload *)sqlite3_aggregate_context(context, sizeof(cloudsync_update_payload));
2666+
if (!payload || payload->count == 0) return;
2667+
2668+
// retrieve context
2669+
sqlite3 *db = sqlite3_context_db_handle(context);
2670+
cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context);
2671+
2672+
// lookup table
2673+
const char *table_name = (const char *)sqlite3_value_text(payload->table_name);
2674+
cloudsync_table_context *table = table_lookup(data, table_name);
2675+
if (!table) {
2676+
dbutils_context_result_error(context, "Unable to retrieve table name %s in cloudsync_update.", table_name);
2677+
return;
2678+
}
2679+
2680+
// compute the next database version for tracking changes
2681+
sqlite3_int64 db_version = db_version_next(db, data, CLOUDSYNC_VALUE_NOTSET);
2682+
int rc = SQLITE_OK;
2683+
2684+
// Check if the primary key(s) have changed
25752685
bool prikey_changed = false;
2576-
for (int i=1; i<=table->npks; ++i) {
2577-
if (dbutils_value_compare(argv[i], argv[i+table->npks]) != 0) {
2686+
for (int i=0; i<table->npks; ++i) {
2687+
if (dbutils_value_compare(payload->old_values[i], payload->new_values[i]) != 0) {
25782688
prikey_changed = true;
25792689
break;
25802690
}
25812691
}
2582-
2692+
25832693
// encode the NEW primary key values into a buffer (used later for indexing)
25842694
char buffer[1024];
25852695
char buffer2[1024];
25862696
size_t pklen = sizeof(buffer);
25872697
size_t oldpklen = sizeof(buffer2);
25882698
char *oldpk = NULL;
25892699

2590-
char *pk = pk_encode_prikey(&argv[1], table->npks, buffer, &pklen);
2700+
char *pk = pk_encode_prikey(payload->new_values, table->npks, buffer, &pklen);
25912701
if (!pk) {
25922702
sqlite3_result_error(context, "Not enough memory to encode the primary key(s).", -1);
25932703
return;
@@ -2599,7 +2709,7 @@ void cloudsync_update (sqlite3_context *context, int argc, sqlite3_value **argv)
25992709
// 2. create a new row (NEW primary key)
26002710

26012711
// encode the OLD primary key into a buffer
2602-
oldpk = pk_encode_prikey(&argv[1+table->npks], table->npks, buffer2, &oldpklen);
2712+
oldpk = pk_encode_prikey(payload->old_values, table->npks, buffer2, &oldpklen);
26032713
if (!oldpk) {
26042714
if (pk != buffer) cloudsync_memory_free(pk);
26052715
sqlite3_result_error(context, "Not enough memory to encode the primary key(s).", -1);
@@ -2626,65 +2736,23 @@ void cloudsync_update (sqlite3_context *context, int argc, sqlite3_value **argv)
26262736
}
26272737

26282738
// compare NEW and OLD values (excluding primary keys) to handle column updates
2629-
// starting index for column values
2630-
int index = 1 + (table->npks * 2);
26312739
for (int i=0; i<table->ncols; i++) {
2632-
if (dbutils_value_compare(argv[i+index], argv[i+index+1]) != 0) {
2740+
int col_index = table->npks + i; // Regular columns start after primary keys
2741+
2742+
if (dbutils_value_compare(payload->old_values[col_index], payload->new_values[col_index]) != 0) {
26332743
// if a column value has changed, mark it as updated in the metadata
26342744
// columns are in cid order
26352745
rc = local_mark_insert_or_update_meta(db, table, pk, pklen, table->col_name[i], db_version, BUMP_SEQ(data));
26362746
if (rc != SQLITE_OK) goto cleanup;
26372747
}
2638-
++index;
26392748
}
26402749

26412750
cleanup:
26422751
if (rc != SQLITE_OK) sqlite3_result_error(context, sqlite3_errmsg(db), -1);
26432752
if (pk != buffer) cloudsync_memory_free(pk);
26442753
if (oldpk && (oldpk != buffer2)) cloudsync_memory_free(oldpk);
2645-
}
2646-
2647-
void cloudsync_delete (sqlite3_context *context, int argc, sqlite3_value **argv) {
2648-
DEBUG_FUNCTION("cloudsync_delete %s", sqlite3_value_text(argv[0]));
2649-
// debug_values(argc-1, &argv[1]);
2650-
2651-
// retrieve context
2652-
sqlite3 *db = sqlite3_context_db_handle(context);
2653-
cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context);
2654-
2655-
// lookup table
2656-
const char *table_name = (const char *)sqlite3_value_text(argv[0]);
2657-
cloudsync_table_context *table = table_lookup(data, table_name);
2658-
if (!table) {
2659-
dbutils_context_result_error(context, "Unable to retrieve table name %s in cloudsync_delete.", table_name);
2660-
return;
2661-
}
2662-
2663-
// compute the next database version for tracking changes
2664-
sqlite3_int64 db_version = db_version_next(db, data, CLOUDSYNC_VALUE_NOTSET);
2665-
int rc = SQLITE_OK;
2666-
2667-
// encode the primary key values into a buffer
2668-
char buffer[1024];
2669-
size_t pklen = sizeof(buffer);
2670-
char *pk = pk_encode_prikey(&argv[1], table->npks, buffer, &pklen);
2671-
if (!pk) {
2672-
sqlite3_result_error(context, "Not enough memory to encode the primary key(s).", -1);
2673-
return;
2674-
}
2675-
2676-
// mark the row as deleted by inserting a delete sentinel into the metadata
2677-
rc = local_mark_delete_meta(db, table, pk, pklen, db_version, BUMP_SEQ(data));
2678-
if (rc != SQLITE_OK) goto cleanup;
2679-
2680-
// remove any metadata related to the old rows associated with this primary key
2681-
rc = local_drop_meta(db, table, pk, pklen);
2682-
if (rc != SQLITE_OK) goto cleanup;
26832754

2684-
cleanup:
2685-
if (rc != SQLITE_OK) sqlite3_result_error(context, sqlite3_errmsg(db), -1);
2686-
// free memory if the primary key was dynamically allocated
2687-
if (pk != buffer) cloudsync_memory_free(pk);
2755+
cloudsync_update_payload_free(payload);
26882756
}
26892757

26902758
// MARK: -
@@ -3274,7 +3342,7 @@ int cloudsync_register (sqlite3 *db, char **pzErrMsg) {
32743342
rc = dbutils_register_function(db, "cloudsync_insert", cloudsync_insert, -1, pzErrMsg, ctx, NULL);
32753343
if (rc != SQLITE_OK) return rc;
32763344

3277-
rc = dbutils_register_function(db, "cloudsync_update", cloudsync_update, -1, pzErrMsg, ctx, NULL);
3345+
rc = dbutils_register_aggregate(db, "cloudsync_update", cloudsync_update_step, cloudsync_update_final, 3, pzErrMsg, ctx, NULL);
32783346
if (rc != SQLITE_OK) return rc;
32793347

32803348
rc = dbutils_register_function(db, "cloudsync_delete", cloudsync_delete, -1, pzErrMsg, ctx, NULL);

‎src/cloudsync.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
extern "C" {
2121
#endif
2222

23-
#define CLOUDSYNC_VERSION "0.8.25"
23+
#define CLOUDSYNC_VERSION "0.8.26"
2424

2525
int sqlite3_cloudsync_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi);
2626
int cloudsync_autoinit (void);

‎src/dbutils.c‎

Lines changed: 44 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -541,25 +541,55 @@ int dbutils_check_triggers (sqlite3 *db, const char *table, table_algo algo) {
541541
if (!trigger_name) goto finalize;
542542

543543
if (!dbutils_trigger_exists(db, trigger_name)) {
544-
char *sql = cloudsync_memory_mprintf("SELECT group_concat('NEW.\"' || format('%%w', name) || '\"', ',') || ',' || group_concat('OLD.\"' || format('%%w', name) || '\"', ',') FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk;", table);
545-
if (!sql) goto finalize;
544+
// Generate VALUES clause for all columns using a CTE to avoid compound SELECT limits
545+
// First, get all primary key columns in order
546+
char *pk_values_sql = cloudsync_memory_mprintf(
547+
"SELECT group_concat('('||quote('%q')||', NEW.\"' || format('%%w', name) || '\", OLD.\"' || format('%%w', name) || '\")', ', ') "
548+
"FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk;",
549+
table, table);
550+
if (!pk_values_sql) goto finalize;
546551

547-
char *pkclause = dbutils_text_select(db, sql);
548-
char *pkvalues = (pkclause) ? pkclause : "NEW.rowid,OLD.rowid";
549-
cloudsync_memory_free(sql);
552+
char *pk_values_list = dbutils_text_select(db, pk_values_sql);
553+
cloudsync_memory_free(pk_values_sql);
550554

551-
sql = cloudsync_memory_mprintf("SELECT group_concat('NEW.\"' || format('%%w', name) || '\"' || ', OLD.\"' || format('%%w', name) || '\"', ',') FROM pragma_table_info('%q') WHERE pk=0 ORDER BY cid;", table);
552-
if (!sql) goto finalize;
553-
char *colvalues = dbutils_text_select(db, sql);
554-
cloudsync_memory_free(sql);
555+
// Then get all regular columns in order
556+
char *col_values_sql = cloudsync_memory_mprintf(
557+
"SELECT group_concat('('||quote('%q')||', NEW.\"' || format('%%w', name) || '\", OLD.\"' || format('%%w', name) || '\")', ', ') "
558+
"FROM pragma_table_info('%q') WHERE pk=0 ORDER BY cid;",
559+
table, table);
560+
if (!col_values_sql) goto finalize;
555561

556-
if (colvalues == NULL) {
557-
sql = cloudsync_memory_mprintf("CREATE TRIGGER \"%w\" AFTER UPDATE ON \"%w\" %s BEGIN SELECT cloudsync_update('%q',%s); END", trigger_name, table, trigger_when, table, pkvalues);
562+
char *col_values_list = dbutils_text_select(db, col_values_sql);
563+
cloudsync_memory_free(col_values_sql);
564+
565+
// Build the complete VALUES query
566+
char *values_query;
567+
if (col_values_list && strlen(col_values_list) > 0) {
568+
// Table has both primary keys and regular columns
569+
values_query = cloudsync_memory_mprintf(
570+
"WITH column_data(table_name, new_value, old_value) AS (VALUES %s, %s) "
571+
"SELECT table_name, new_value, old_value FROM column_data",
572+
pk_values_list, col_values_list);
573+
cloudsync_memory_free(col_values_list);
558574
} else {
559-
sql = cloudsync_memory_mprintf("CREATE TRIGGER \"%w\" AFTER UPDATE ON \"%w\" %s BEGIN SELECT cloudsync_update('%q',%s,%s); END", trigger_name, table, trigger_when, table, pkvalues, colvalues);
560-
cloudsync_memory_free(colvalues);
575+
// Table has only primary keys
576+
values_query = cloudsync_memory_mprintf(
577+
"WITH column_data(table_name, new_value, old_value) AS (VALUES %s) "
578+
"SELECT table_name, new_value, old_value FROM column_data",
579+
pk_values_list);
561580
}
562-
if (pkclause) cloudsync_memory_free(pkclause);
581+
582+
if (pk_values_list) cloudsync_memory_free(pk_values_list);
583+
if (!values_query) goto finalize;
584+
585+
// Create the trigger with aggregate function
586+
char *sql = cloudsync_memory_mprintf(
587+
"CREATE TRIGGER \"%w\" AFTER UPDATE ON \"%w\" %s BEGIN "
588+
"SELECT cloudsync_update(table_name, new_value, old_value) FROM (%s); "
589+
"END",
590+
trigger_name, table, trigger_when, values_query);
591+
592+
cloudsync_memory_free(values_query);
563593
if (!sql) goto finalize;
564594

565595
rc = sqlite3_exec(db, sql, NULL, NULL, NULL);

0 commit comments

Comments
 (0)