diff --git a/bdb/bdb_api.h b/bdb/bdb_api.h index 8de50afbc8..42559384ce 100644 --- a/bdb/bdb_api.h +++ b/bdb/bdb_api.h @@ -1755,11 +1755,10 @@ int bdb_newsc_del_redo_genid(tran_type *t, const char *tablename, uint64_t genid int bdb_newsc_del_all_redo_genids(tran_type *t, const char *tablename, int *bdberr); -int bdb_set_high_genid(tran_type *input_trans, const char *tablename, - unsigned long long genid, int *bdberr); -int bdb_set_high_genid_stripe(tran_type *input_trans, const char *db_name, - int stripe, unsigned long long genid, - int *bdberr); +int bdb_set_high_genid(tran_type *input_trans, const char *tablename, unsigned long long genid, int *bdberr, + const char *f, int l); +int bdb_set_high_genid_stripe(tran_type *input_trans, const char *db_name, int stripe, unsigned long long genid, + int *bdberr, const char *f, int l); int bdb_clear_high_genid(tran_type *input_trans, const char *db_name, int num_stripes, int *bdberr); int bdb_get_high_genid(const char *db_name, int stripe, diff --git a/bdb/llmeta.c b/bdb/llmeta.c index 9333b5e6db..ccd567fdb2 100644 --- a/bdb/llmeta.c +++ b/bdb/llmeta.c @@ -5150,16 +5150,23 @@ int bdb_clear_high_genid( /* determines what stripe the genid is part of and calls * bdb_set_high_genid_int */ -int bdb_set_high_genid(tran_type *input_trans, const char *db_name, - unsigned long long genid, int *bdberr) +int bdb_set_high_genid(tran_type *input_trans, const char *db_name, unsigned long long genid, int *bdberr, + const char *f, int l) { +#ifdef DEBUG_LLMETA + fprintf(stderr, "%s: (%s:%d) setting %s stripe %d to %llx (%llu)\n", __func__, f, l, db_name, + get_dtafile_from_genid(genid), genid, genid); +#endif return bdb_set_high_genid_int(input_trans, db_name, get_dtafile_from_genid(genid), genid, bdberr); } -int bdb_set_high_genid_stripe(tran_type *input_trans, const char *db_name, - int stripe, unsigned long long genid, int *bdberr) +int bdb_set_high_genid_stripe(tran_type *input_trans, const char *db_name, int stripe, unsigned long long genid, + int *bdberr, const char *f, int l) { +#ifdef DEBUG_LLMETA + fprintf(stderr, "%s: (%s:%d) setting %s stripe %d to %llx (%llu)\n", __func__, f, l, db_name, stripe, genid, genid); +#endif return bdb_set_high_genid_int(input_trans, db_name, stripe, genid, bdberr); } @@ -10510,7 +10517,7 @@ static int bdb_process_each_table_entry(bdb_state_type *bdb_state, key_struct.dbname_len = strlen(key_struct.dbname) + 1 /* NULL byte */; if (key_struct.dbname_len > LLMETA_TBLLEN) { - fprintf(stderr, "%s: db_name is too long\n", __func__); + logmsg(LOGMSG_ERROR, "%s: db_name is too long\n", __func__); *bdberr = BDBERR_BADARGS; return -1; } diff --git a/db/comdb2.h b/db/comdb2.h index cd0c44c9b9..3af6f029f8 100644 --- a/db/comdb2.h +++ b/db/comdb2.h @@ -1768,7 +1768,6 @@ extern int gbl_reset_queue_cursor; extern int gbl_readonly; extern int gbl_readonly_sc; extern int gbl_init_single_meta; -extern unsigned long long gbl_sc_genids[MAXDTASTRIPE]; extern int gbl_sc_usleep; extern int gbl_sc_wrusleep; extern int gbl_sc_last_writer_time; @@ -3645,8 +3644,8 @@ const char *sc_tag_change_subtype_text(sc_tag_change_subtype); int cmp_index_int(struct schema *oldix, struct schema *newix, char *descr, size_t descrlen); int get_dbtable_idx_by_name(const char *tablename); -int open_temp_db_resume(struct ireq *iq, struct dbtable *db, char *prefix, int resume, - int temp, tran_type *tran); +int open_temp_db_resume(struct ireq *iq, struct dbtable *db, char *tablename, int resume); +int open_temp_newdb_resume(struct ireq *iq, struct dbtable *db, int resume); int find_constraint(struct dbtable *db, constraint_t *ct); /* END OF SCHEMACHANGE DECLARATIONS*/ diff --git a/db/osqlcomm.c b/db/osqlcomm.c index 034463c021..035eba2ee0 100644 --- a/db/osqlcomm.c +++ b/db/osqlcomm.c @@ -6024,6 +6024,35 @@ static inline int is_write_request(int type) void free_cached_idx(uint8_t **cached_idx); +int dbtable_get_highest_genid(struct dbtable *table, unsigned long long *genids) +{ + int rc; + void *rec; + int i; + int orglen = MAXLRL; + int bdberr; + + rec = alloca(orglen); + + /* get max genid for each stripe */ + for (i = 0; i < table->dtastripe; ++i) { + uint8_t ver; + int dtalen = orglen; + + rc = bdb_find_newest_genid(table->handle, NULL, i, rec, &dtalen, dtalen, &genids[i], &ver, &bdberr); + if (rc == IX_FND) + logmsg(LOGMSG_INFO, "%s: LOOKING FOR %s STRIPE %d found genid %llx (%lld)\n", __func__, table->tablename, i, + genids[i], genids[i]); + else if (rc == 1) + logmsg(LOGMSG_INFO, "%s: LOOKING FOR %s STRIPE %d empty stripe, genid will be 0\n", __func__, + table->tablename, i); + else if (rc < 0 || bdberr != BDBERR_NOERROR) { + return -1; + } + } + return 0; +} + int gbl_disable_tpsc_tblvers = 0; static int start_schema_change_tran_wrapper(const char *tblname, timepart_view_t **pview, @@ -6116,8 +6145,18 @@ static int start_schema_change_tran_wrapper(const char *tblname, rc = populate_db_with_alt_schema(thedb, sc->newdb, sc->newcsc2, &err); if (rc) { logmsg(LOGMSG_ERROR, "%s: populate_db_with_alt_schema failed with rc %d %s\n", __func__, rc, err.errstr); + sc_errf(sc, "%s: populate_db_with_alt_schema failed with rc %d %s\n", __func__, rc, err.errstr); return VIEW_ERR_SC; } + + /* we also need to retrieve the highest genid */ + if (arg->indx > 0) { + rc = dbtable_get_highest_genid(sc->newdb, arg->retros->cs[arg->indx - 1].resume_genids); + if (rc) { + sc_errf(sc, "%s: failed to find newest genid for shard %s\n", __func__, sc->tablename); + return VIEW_ERR_SC; + } + } } if (arg->lockless) { @@ -6352,6 +6391,7 @@ static int _process_partitioning_retro(timepart_sc_arg_t *arg) struct schema_change_type *sc = arg->s; struct errstat err = {0}; int rc = 0; + int ii; /* determine retroactive time boundaries for the shards */ int len = sizeof(struct timepart_retro) + @@ -6370,7 +6410,7 @@ static int _process_partitioning_retro(timepart_sc_arg_t *arg) sc->partition.u.tpt.retention * sizeof(int)); retros->cs = (timepart_retro_ctr_t *)((char *)retros + sizeof(struct timepart_retro) + sc->partition.u.tpt.retention * (sizeof(int) + sizeof(int *))); - for (int ii = 0; ii < sc->partition.u.tpt.retention; ii++) { + for (ii = 0; ii < sc->partition.u.tpt.retention; ii++) { pthread_mutex_init(&retros->cs[ii].mtx, 0); } rc = timepart_populate_timelimits(sc->newpartition, retros, &err); @@ -6405,6 +6445,20 @@ static int _process_partitioning_retro(timepart_sc_arg_t *arg) } retros->ss[sc->partition.u.tpt.retention - 1] = arg->s; + /* set the maximum genid for each stripe */ + for (int stripe = 0; stripe < sc->newdb->dtastripe; stripe++) { + for (ii = 0; ii < sc->partition.u.tpt.retention; ii++) { + unsigned long long genid = retros->cs[ii].resume_genids[stripe]; + if (genid > retros->resume_genids[stripe]) { + logmsg(LOGMSG_INFO, + "%s: increased stripe %d resume genid from %llx (%lld) to %llx (%lld) per shard %s\n", __func__, + stripe, retros->resume_genids[stripe], retros->resume_genids[stripe], genid, genid, + sc->tablename); + retros->resume_genids[stripe] = genid; + } + } + } + /* alter existing shard */ arg->indx = 0; arg->pos = FIRST_SHARD | LAST_SHARD; @@ -6416,6 +6470,8 @@ static int _process_partitioning_retro(timepart_sc_arg_t *arg) db->sharding_func = timepart_retro_route; sc->newpartition = newpartition; sc->force_rebuild = 1; + /* run this async, this can be in upgrade thread */ + sc->nothrevent = 0; for (int jj = 0; jj < retros->n; jj++) { logmsg(LOGMSG_USER, "PARTITION %s shard %d time %u name %s\n", arg->part_name, jj, retros->limits[jj], (jj < (retros->n - 1)) ? retros->ss[jj]->tablename : arg->part_name); diff --git a/db/views.h b/db/views.h index d79758601b..dbca76ffd4 100644 --- a/db/views.h +++ b/db/views.h @@ -77,6 +77,7 @@ enum views_trigger_op { typedef struct timepart_retro_ctr { pthread_mutex_t mtx; int counter; + unsigned long long resume_genids[MAXDTASTRIPE]; } timepart_retro_ctr_t; typedef struct timepart_retro { @@ -84,6 +85,7 @@ typedef struct timepart_retro { int *limits; struct schema_change_type **ss; timepart_retro_ctr_t *cs; + unsigned long long resume_genids[MAXDTASTRIPE]; } timepart_retro_t; enum shard_pos { /* keep them bits */ diff --git a/schemachange/sc_add_table.c b/schemachange/sc_add_table.c index e636572bea..94789bbf92 100644 --- a/schemachange/sc_add_table.c +++ b/schemachange/sc_add_table.c @@ -128,7 +128,13 @@ int add_table_to_environment(char *table, const char *csc2, goto err; } - if ((rc = get_db_handle(newdb, trans))) { + if (s && s->resume && s->partition.type == PARTITION_ADD_TIMED_RETRO) { + /* this is adding a shard, we need to try to open an existing shard, which may have data */ + if ((rc = open_temp_db_resume(iq, newdb, newdb->tablename, s->resume))) { + sc_errf(s, "Failed to open shard %s\n", newdb->tablename); + reqerrstr(iq, ERR_SC, "Failed to open shard %s\n", newdb->tablename); + } + } else if ((rc = get_db_handle(newdb, trans))) { if (rc == BDBERR_EXCEEDED_BLOBS){ sc_errf(s, "Maximum number of vutf8/blob fields exceeded\n"); reqerrstr(iq, ERR_SC, "Maximum number of vutf8/blob fields exceeded\n"); @@ -136,6 +142,8 @@ int add_table_to_environment(char *table, const char *csc2, sc_errf(s, "Maximum number of indexes exceeded\n"); reqerrstr(iq, ERR_SC, "Maximum number of indexes exceeded\n"); } + } + if (rc) { rc = SC_BDB_ERROR; goto err; } diff --git a/schemachange/sc_alter_table.c b/schemachange/sc_alter_table.c index 55243f5b3b..16f90c70e5 100644 --- a/schemachange/sc_alter_table.c +++ b/schemachange/sc_alter_table.c @@ -399,7 +399,6 @@ int do_alter_table(struct ireq *iq, struct schema_change_type *s, int datacopy_odh = 0; int changed; int i; - char new_prefix[32]; struct scinfo scinfo; struct errstat err = {0}; @@ -522,18 +521,7 @@ int do_alter_table(struct ireq *iq, struct schema_change_type *s, print_schemachange_info(s, db, newdb); /*************** open tables ********************************************/ - - /* create temporary tables. to try to avoid strange issues always - * use a unqiue prefix. this avoids multiple histories for these - * new. files in our logs. - * - * since the prefix doesn't matter and bdb needs to be able to unappend - * it, we let bdb choose the prefix */ - /* ignore failures, there shouln't be any and we'd just have a - * truncated prefix anyway */ - bdb_get_new_prefix(new_prefix, sizeof(new_prefix), &bdberr); - - rc = open_temp_db_resume(iq, newdb, new_prefix, s->resume, 0, tran); + rc = open_temp_newdb_resume(iq, newdb, s->resume); if (rc) { if (rc == BDBERR_EXCEEDED_BLOBS) { sc_errf(s, "Maximum number of vutf8/blob fields exceeded\n"); @@ -609,6 +597,29 @@ int do_alter_table(struct ireq *iq, struct schema_change_type *s, return -1; } + if (s->resume && s->partition.type == PARTITION_ADD_TIMED_RETRO) { + /* we have more shards here where we put data, + * update newdb->sc_genids with the max of all shard stripes + */ + assert(db->sharding_arg); + assert(db->sharding_func); + for (i = 0; i < newdb->dtastripe; i++) { + for (int shard = 0; shard < db->sharding_arg->n - 1; shard++) { + if (newdb->sc_genids[i] < db->sharding_arg->cs[shard].resume_genids[i]) { + logmsg(LOGMSG_INFO, "%s updating %s stripe %d sc_genid from %llx (%lld) to %llx (%lld) using %s\n", + __func__, s->tablename, i, newdb->sc_genids[i], newdb->sc_genids[i], + db->sharding_arg->cs[shard].resume_genids[i], db->sharding_arg->cs[shard].resume_genids[i], + db->sharding_arg->ss[shard]->tablename); + newdb->sc_genids[i] = db->sharding_arg->cs[shard].resume_genids[i]; + } + } + } + } + for (i = 0; i < newdb->dtastripe; i++) { + logmsg(LOGMSG_INFO, "%s: %s stripe %d result resume genid %llx (%lld)\n", __func__, s->tablename, i, + newdb->sc_genids[i], newdb->sc_genids[i]); + } + Pthread_rwlock_wrlock(&db->sc_live_lk); db->sc_from = s->db = db; db->sc_to = s->newdb = newdb; diff --git a/schemachange/sc_fastinit_table.c b/schemachange/sc_fastinit_table.c index a679890343..5c3ab1be94 100644 --- a/schemachange/sc_fastinit_table.c +++ b/schemachange/sc_fastinit_table.c @@ -56,9 +56,7 @@ int do_fastinit(struct ireq *iq, struct schema_change_type *s, tran_type *tran) struct dbtable *db; struct dbtable *newdb; int rc = 0; - int bdberr = 0; int datacopy_odh = 0; - char new_prefix[32]; struct scinfo scinfo; struct errstat err = {0}; @@ -103,22 +101,12 @@ int do_fastinit(struct ireq *iq, struct schema_change_type *s, tran_type *tran) Pthread_mutex_unlock(&csc2_subsystem_mtx); - /* create temporary tables. to try to avoid strange issues always - * use a unqiue prefix. this avoids multiple histories for these - * new. files in our logs. - * - * since the prefix doesn't matter and bdb needs to be able to unappend - * it, we let bdb choose the prefix */ - /* ignore failures, there shouln't be any and we'd just have a - * truncated prefix anyway */ - bdb_get_new_prefix(new_prefix, sizeof(new_prefix), &bdberr); - int local_lock = 0; if (!iq->sc_locked) { local_lock = 1; wrlock_schema_lk(); } - rc = open_temp_db_resume(iq, newdb, new_prefix, 0, 0, tran); + rc = open_temp_newdb_resume(iq, newdb, 0); if (local_lock) unlock_schema_lk(); if (rc) { diff --git a/schemachange/sc_logic.c b/schemachange/sc_logic.c index 75cdb3f161..e14ceb725e 100644 --- a/schemachange/sc_logic.c +++ b/schemachange/sc_logic.c @@ -1382,13 +1382,15 @@ int resume_schema_change(void) /****************** Table functions ***********************************/ /****************** Functions down here will likely be moved elsewhere *****/ -/* this assumes threads are not active in db */ -int open_temp_db_resume(struct ireq *iq, struct dbtable *db, char *prefix, int resume, - int temp, tran_type *tran) +int open_temp_newdb_resume(struct ireq *iq, struct dbtable *db, int resume) { char *tmpname; int bdberr; int nbytes; + char prefix[32]; + int rc; + + bdb_get_new_prefix(prefix, sizeof(prefix), &bdberr); nbytes = snprintf(NULL, 0, "%s%s", prefix, db->tablename); if (nbytes <= 0) nbytes = 2; @@ -1396,28 +1398,34 @@ int open_temp_db_resume(struct ireq *iq, struct dbtable *db, char *prefix, int r tmpname = malloc(nbytes); snprintf(tmpname, nbytes, "%s%s", prefix, db->tablename); + rc = open_temp_db_resume(iq, db, tmpname, resume); + + free(tmpname); + + return rc; +} + +int open_temp_db_resume(struct ireq *iq, struct dbtable *db, char *tablename, int resume) +{ + int bdberr; + db->handle = NULL; /* open existing temp db if it's there (ie we're resuming after a master * switch) */ if (resume) { - db->handle = bdb_open_more( - tmpname, db->dbenv->basedir, db->lrl, db->nix, - (short *)db->ix_keylen, db->ix_dupes, db->ix_recnums, - db->ix_datacopy, db->ix_datacopylen, db->ix_collattr, db->ix_nullsallowed, - db->numblobs + 1, /* one main record + the blobs blobs */ - db->dbenv->bdb_env, &bdberr); + db->handle = bdb_open_more(tablename, db->dbenv->basedir, db->lrl, db->nix, (short *)db->ix_keylen, + db->ix_dupes, db->ix_recnums, db->ix_datacopy, db->ix_datacopylen, db->ix_collattr, + db->ix_nullsallowed, db->numblobs + 1, /* one main record + the blobs blobs */ + db->dbenv->bdb_env, &bdberr); if (db->handle) logmsg(LOGMSG_INFO, "Found existing tempdb: %s, attempting to resume an in " "progress schema change\n", - tmpname); + tablename); else { - logmsg(LOGMSG_ERROR, - "Didn't find existing tempdb: %s, aborting schema change\n", - tmpname); - free(tmpname); + logmsg(LOGMSG_ERROR, "Didn't find existing tempdb: %s, aborting schema change\n", tablename); return -1; } } @@ -1425,49 +1433,38 @@ int open_temp_db_resume(struct ireq *iq, struct dbtable *db, char *prefix, int r if (!db->handle) /* did not/could not open existing one, creating new one */ { int rc; - tran_type * tmp_tran = tran; + tran_type *tran = NULL; retry: - if (!tmp_tran) { - rc = trans_start(iq, NULL, &tmp_tran); - if (rc) - return -1; - } + rc = trans_start(iq, NULL, &tran); + if (rc) + return -1; - db->handle = bdb_create_tran( - tmpname, db->dbenv->basedir, db->lrl, db->nix, - (short *)db->ix_keylen, db->ix_dupes, db->ix_recnums, - db->ix_datacopy, db->ix_datacopylen, db->ix_collattr, db->ix_nullsallowed, - db->numblobs + 1, /* one main record + the blobs blobs */ - db->dbenv->bdb_env, temp, &bdberr, tmp_tran); + db->handle = bdb_create_tran(tablename, db->dbenv->basedir, db->lrl, db->nix, (short *)db->ix_keylen, + db->ix_dupes, db->ix_recnums, db->ix_datacopy, db->ix_datacopylen, db->ix_collattr, + db->ix_nullsallowed, db->numblobs + 1, /* one main record + the blobs blobs */ + db->dbenv->bdb_env, 0, &bdberr, tran); if (db->handle == NULL) { - if (tmp_tran != tran) { - trans_abort(iq, tmp_tran); - tmp_tran = NULL; - if (bdberr == BDBERR_DEADLOCK) { - logmsg(LOGMSG_WARN, "%s: retrying on BDBERR_DEADLOCK\n", __func__); - goto retry; - } + trans_abort(iq, tran); + tran = NULL; + if (bdberr == BDBERR_DEADLOCK) { + logmsg(LOGMSG_WARN, "%s: retrying on BDBERR_DEADLOCK\n", __func__); + goto retry; } - logmsg(LOGMSG_ERROR, "%s: failed to open %s, rcode %d\n", __func__, - tmpname, bdberr); + logmsg(LOGMSG_ERROR, "%s: failed to open %s, rcode %d\n", __func__, tablename, bdberr); - free(tmpname); return bdberr; } - if (tmp_tran != tran) { - rc = trans_commit_nowait(iq, tmp_tran, gbl_myhostname); - if (rc) - return -1; - } + rc = trans_commit_nowait(iq, tran, gbl_myhostname); + if (rc) + return -1; } /* clone the blobstripe genid. this will definately be needed in the * future when we don't change genids on schema change, but right now * isn't really needed. */ bdb_set_blobstripe_genid(db->handle, db->blobstripe_genid); - free(tmpname); return 0; } diff --git a/schemachange/sc_records.c b/schemachange/sc_records.c index b57506d208..66b2fab6a9 100644 --- a/schemachange/sc_records.c +++ b/schemachange/sc_records.c @@ -195,6 +195,7 @@ static inline void lkcounter_check(struct convert_record_data *data, int now) * If the schema change is not resuming it sets them all to zero * If success it returns 0, if failure it returns <0 */ int gbl_debug_stall_in_oplog_seed = 0; + int init_sc_genids(struct dbtable *db, struct schema_change_type *s) { void *rec; @@ -275,8 +276,14 @@ int init_sc_genids(struct dbtable *db, struct schema_change_type *s) rc = bdb_find_newest_genid(db->handle, NULL, stripe, rec, &dtalen, dtalen, &sc_genids[stripe], &ver, &bdberr); - if (rc == 1) + if (rc == IX_FND) + logmsg(LOGMSG_DEBUG, "%s: LOOKING FOR %s STRIPE %d found genid %llx (%lld)\n", __func__, db->tablename, + stripe, sc_genids[stripe], sc_genids[stripe]); + if (rc == 1) { + logmsg(LOGMSG_DEBUG, "%s: LOOKING FOR %s STRIPE %d reset to genid 0\n", __func__, db->tablename, + stripe); sc_genids[stripe] = 0ULL; + } } else rc = bdb_get_high_genid(db->tablename, stripe, &sc_genids[stripe], &bdberr); @@ -812,14 +819,12 @@ static int convert_record(struct convert_record_data *data) rc = 0; if (usellmeta && !is_dta_being_rebuilt(data->to->plan)) { int bdberr; - rc = bdb_set_high_genid_stripe(NULL, data->to->tablename, - data->stripe, -1ULL, &bdberr); + rc = bdb_set_high_genid_stripe(NULL, data->to->tablename, data->stripe, -1ULL, &bdberr, __func__, + __LINE__); if (rc != 0) rc = -1; // convert_record expects -1 + sc_printf(data->s, "[%s] finished stripe %d, setting genid %llx, rc %d\n", data->from->tablename, + data->stripe, data->sc_genids[data->stripe], rc); } - sc_printf(data->s, - "[%s] finished stripe %d, setting genid %llx, rc %d\n", - data->from->tablename, data->stripe, - data->sc_genids[data->stripe], rc); return rc; } else if (rc == RC_INTERNAL_RETRY) { trans_abort(&data->iq, data->trans); @@ -1061,6 +1066,10 @@ static int convert_record(struct convert_record_data *data) 0, /* blkpos */ addflags, 0); + if (rc) + logmsg(LOGMSG_ERROR, "Failed to add record %llx (%lld) in migration %s->%s rc %d\n", ngenid, ngenid, + tbl->tablename, data->iq.usedb->tablename, rc); + data->iq.usedb = tbl; if (rc) @@ -1073,8 +1082,7 @@ static int convert_record(struct convert_record_data *data) (data->nrecs % BDB_ATTR_GET(thedb->bdb_attr, INDEXREBUILD_SAVE_EVERY_N)) == 0) { int bdberr; - rc = bdb_set_high_genid(data->trans, data->to->tablename, genid, - &bdberr); + rc = bdb_set_high_genid(data->trans, data->to->tablename, genid, &bdberr, __func__, __LINE__); if (rc != 0) { if (bdberr == BDBERR_DEADLOCK) rc = RC_INTERNAL_RETRY; @@ -2705,8 +2713,7 @@ static int live_sc_redo_add(struct convert_record_data *data, DB_LOGC *logc, if (!is_dta_being_rebuilt(data->to->plan)) { int bdberr; - rc = bdb_set_high_genid(data->trans, data->to->tablename, genid, - &bdberr); + rc = bdb_set_high_genid(data->trans, data->to->tablename, genid, &bdberr, __func__, __LINE__); if (rc != 0) { if (bdberr == BDBERR_DEADLOCK) rc = RC_INTERNAL_RETRY; diff --git a/schemachange/sc_struct.c b/schemachange/sc_struct.c index 00dc628ecb..080ad37d66 100644 --- a/schemachange/sc_struct.c +++ b/schemachange/sc_struct.c @@ -1935,6 +1935,7 @@ struct schema_change_type *clone_schemachange_type(struct schema_change_type *sc newsc->timepartition_version = sc->timepartition_version; newsc->partition = sc->partition; newsc->usedbtablevers = sc->usedbtablevers; + newsc->resume = sc->resume; if (!p_buf) { free_schema_change_type(newsc); @@ -1943,5 +1944,6 @@ struct schema_change_type *clone_schemachange_type(struct schema_change_type *sc } free(buf); + return newsc; } diff --git a/schemachange/schemachange.c b/schemachange/schemachange.c index 1efa51347b..9039daafea 100644 --- a/schemachange/schemachange.c +++ b/schemachange/schemachange.c @@ -894,7 +894,6 @@ static int add_table_for_recovery(struct ireq *iq, struct schema_change_type *s) { struct dbtable *db; struct dbtable *newdb; - int bdberr; int rc; db = get_dbtable_by_name(s->tablename); @@ -914,8 +913,6 @@ static int add_table_for_recovery(struct ireq *iq, struct schema_change_type *s) return -1; } - char new_prefix[32]; - if (s->headers != db->odh) { s->header_change = s->force_dta_rebuild = s->force_blob_rebuild = 1; } @@ -943,9 +940,7 @@ static int add_table_for_recovery(struct ireq *iq, struct schema_change_type *s) abort(); } - bdb_get_new_prefix(new_prefix, sizeof(new_prefix), &bdberr); - - rc = open_temp_db_resume(iq, newdb, new_prefix, 1, 0, NULL); + rc = open_temp_newdb_resume(iq, newdb, 1); if (rc) { backout_schemas(newdb->tablename); abort(); @@ -953,6 +948,7 @@ static int add_table_for_recovery(struct ireq *iq, struct schema_change_type *s) return 0; } + /* Make sure that logical recovery has tables to work with */ int add_schema_change_tables() { diff --git a/tests/timepart_retro.test/lrl.options b/tests/timepart_retro.test/lrl.options index 7806570e26..24539ee0f6 100644 --- a/tests/timepart_retro.test/lrl.options +++ b/tests/timepart_retro.test/lrl.options @@ -2,3 +2,4 @@ table t t.csc2 init_with_genid48 0 partition_retroactively_verbose 1 partition_retroactively_start 1 +multitable_ddl 1 diff --git a/tests/timepart_retro.test/output.log b/tests/timepart_retro.test/output.log index e5cdd37e9b..fb93caa22e 100644 --- a/tests/timepart_retro.test/output.log +++ b/tests/timepart_retro.test/output.log @@ -1,5 +1,6 @@ (rows inserted=60) (rows inserted=60) + (shardname='$0_F64CD191') (shardname='$1_A2620AE4') (count(*)=110) diff --git a/tests/timepart_retro.test/runit b/tests/timepart_retro.test/runit index 47e12e01ed..1490357df0 100755 --- a/tests/timepart_retro.test/runit +++ b/tests/timepart_retro.test/runit @@ -19,15 +19,22 @@ delayed_writes() { $CM "exec procedure sys.cmd.send('debg 500')" $CM "exec procedure sys.cmd.send('ndebg 500')" - echo "Sleeping 10 seconds" + echo "Sleeping 10 seconds for deletes" sleep 10 $C "delete from t where a between 1 and 10 or a between 101 and 110" - echo "Sleeping 10 more seconds" + echo "Sleeping 10 more seconds for updates" sleep 10 $C "update t set a = a + 1000 where a between 11 and 15 or a between 111 and 115" + echo "Sleeping 10 more seconds for inserts" sleep 10 - echo "Sleeping 10 more seconds finally" $C "insert into t(a) select * from generate_series(2001, 2010)" + if [[ -z "${CLUSTER}" ]] ; then + echo "Skipping master downgrade test since we are single node" + else + echo "Sleeping 10 more seconds finally for master downgrade ${master}" + sleep 10 + $CM "exec procedure sys.cmd.send('downgrade')" + fi } echo "Inserting the first set" @@ -70,12 +77,24 @@ echo "Launching delayed writes" delayed_writes & echo "Partitioning the table" -$C "ALTER TABLE T PARTITIONED BY TIME PERIOD 'daily' RETENTION 2 START ${start} RETROACTIVELY" >> $OUT -if (( $? != 0 )) ; then - echo "FAILURE to partition retroactively" - exit 1 +output=`$C "ALTER TABLE T PARTITIONED BY TIME PERIOD 'daily' RETENTION 2 START ${start} RETROACTIVELY"` +echo ${output} >> $OUT +if [[ ! -z "${output}" ]] ; then + echo "Received error ${output}" + if [[ "${output}" == *"Master node downgrading"* ]] ; then + echo "Master downgrading error, ignoring, we'll check resume results" + else + echo "FAILURE to partition retroactively" + exit 1 + fi fi +echo "make sure subprocess is gone" +wait + +echo "waiting 30 seconds to allow schema to finish" +sleep 30 + echo "Checking the shards" $C "select shardname from comdb2_timepartshards" >> $OUT if (( $? != 0 )) ; then