Fix sql logfill

berkdb/txn/txn.c

@@ -1023,6 +1023,7 @@ void comdb2_cheapstack_sym(FILE *f, char *fmt, ...);
 
 extern int gbl_fullrecovery;
 int gbl_endianize_locklist = 1;
+int gbl_emit_gen_commits = 1;
 
 /*
  * __txn_commit --
@@ -1131,7 +1132,7 @@ __txn_commit_int(txnp, flags, ltranid, llid, last_commit_lsn, rlocks, inlks,
 	}
 
     /* don't let full recovery write a (higher) generation: it will force this newly-recovered node to be master on the next election */
-	elect_highest_committed_gen = !gbl_fullrecovery;
+	elect_highest_committed_gen = (!gbl_fullrecovery && gbl_emit_gen_commits);
 	db_rep = dbenv->rep_handle;
 	rep = db_rep->region;
 

db/db_tunables.c

@@ -149,6 +149,7 @@ extern int gbl_round_robin_stripes;
 extern int skip_clear_queue_extents;
 extern int gbl_rep_skip_recovery;
 extern int gbl_retrieve_gen_from_ckp;
+extern int gbl_emit_gen_commits;
 extern int gbl_recovery_ckp;
 extern int gbl_reproduce_ckp_bug;
 extern int gbl_sample_queries;

db/db_tunables.h

@@ -1057,6 +1057,8 @@ REGISTER_TUNABLE("rep_process_txn_trace",
                  NULL, NULL, NULL, NULL);
 REGISTER_TUNABLE("rep_skip_recovery", "Skip recovery if truncate won't unwind a transaction.  (Default: off)",
                  TUNABLE_BOOLEAN, &gbl_rep_skip_recovery, 0, NULL, NULL, NULL, NULL);
+REGISTER_TUNABLE("emit_gen_commits", "Emit commit-records which include cluster generation.  (Default: on)",
+                 TUNABLE_BOOLEAN, &gbl_emit_gen_commits, 0, NULL, NULL, NULL, NULL);
 /* 'retrieve_gen_from_ckp' / 'recovery_ckp' disabled under legacy_defaults until db moves */
 REGISTER_TUNABLE("retrieve_gen_from_ckp", "Retrieve generation from ckp records.  (Default: on)", TUNABLE_BOOLEAN,
                  &gbl_retrieve_gen_from_ckp, 0, NULL, NULL, NULL, NULL);

db/sqllogfill.c

@@ -328,7 +328,20 @@ static void print_record_info(const char *prefix, const char *lsn)
 
 static inline int gen_okay(bdb_state_type *bdb_state, int64_t recgen, u_int32_t mygen)
 {
-    return recgen <= mygen;
+    if (recgen > mygen) {
+        static int64_t cnt = 0;
+        static int lastprint = 0;
+        cnt++;
+        if (gbl_debug_sql_logfill) {
+            int now = comdb2_time_epoch();
+            if (now - lastprint > 0) {
+                logmsg(LOGMSG_USER, "%s: recgen > mygen: recgen=%" PRId64 " mygen=%u cnt=%" PRId64 "\n", __func__,
+                       recgen, mygen, cnt);
+                lastprint = now;
+            }
+        }
+    }
+    return 1;
 }
 
 static int apply_record(bdb_state_type *bdb_state, const char *lsn, void *blob, int blob_len, int64_t genp,
@@ -698,7 +711,7 @@ static void *sql_apply_thread(void *arg)
             BDB_READLOCK(__func__);
             u_int32_t gen;
             bdb_state->dbenv->get_rep_gen(bdb_state->dbenv, &gen);
-            if (copy.gen == gen && (copy.recgen == 0 || gen >= copy.recgen)) {
+            if (copy.gen == gen && (copy.recgen == 0 || gen_okay(bdb_state, copy.recgen, gen))) {
                 rep_apply_caller = "sqllogfill-apply-thread";
                 bdb_state->dbenv->apply_log(bdb_state->dbenv, copy.file, copy.offset, copy.rectype, copy.blob,
                                             copy.blob_len);

tests/noresetgen.test/Makefile

@@ -0,0 +1,10 @@
+ifeq ($(TESTSROOTDIR),)
+  include ../testcase.mk
+else
+  include $(TESTSROOTDIR)/testcase.mk
+endif
+export CHECK_DB_AT_FINISH=0
+ifeq ($(TEST_TIMEOUT),)
+	export TEST_TIMEOUT=10m
+endif
+

tests/noresetgen.test/lrl.options

@@ -0,0 +1,4 @@
+# Use default options - features that preserve generation are enabled by default
+# This test verifies that generation number is preserved and increments across cluster restarts
+
+logmsg level debug

tests/noresetgen.test/runit

@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+
+bash -n "$0" | exit 1
+
+. ${TESTSROOTDIR}/tools/runit_common.sh
+
+. ${TESTSROOTDIR}/tools/cluster_utils.sh
+
+export debug=1
+[[ "$debug" == "1" ]] && set -x
+
+db=$1
+cnt=$(echo $CLUSTER | wc -w)
+
+if [[ -z "$CLUSTER" || $cnt -lt 3 ]]; then
+    echo "This test requires a clustered installation of at least 3 nodes"
+    exit 0
+fi
+
+
+function run_test {
+    echo "=== Starting generation preservation test (default options) ==="
+
+    # Get initial master and generation
+    master=$(get_master)
+    initial_gen=$(get_node_gen $master)
+    echo "Initial master: $master"
+    echo "Initial generation: $initial_gen"
+
+    # Test: Stop and restart cluster 10 times, verify generation increments each time
+    echo "=== Testing generation preservation through cluster restarts ==="
+
+    prev_gen=$initial_gen
+    for i in {1..10}; do
+        # Get current master and generation before restart
+        master=$(get_master)
+        current_gen=$(get_node_gen $master)
+        echo "  Iteration $i: gen=$current_gen, master=$master"
+
+        # Stop entire cluster
+        echo "  Stopping entire cluster"
+        stop_all_nodes
+
+        # Verify all nodes are down
+        for node in $CLUSTER; do
+            $CDB2SQL_EXE ${CDB2_OPTIONS} $db --host $node "select 1" >/dev/null 2>&1
+            if [[ $? -eq 0 ]]; then
+                failexit "Node $node is still up after stop"
+            fi
+        done
+
+        # Restart entire cluster
+        echo "  Restarting entire cluster"
+        start_all_nodes
+        sleep 5
+
+        # Wait for cluster coherency
+        wait_for_cluster
+
+        # Get new master and generation after restart
+        master=$(get_master)
+        new_gen=$(get_node_gen $master)
+        echo "  After restart: gen=$new_gen, new_master=$master"
+
+        # Verify generation incremented
+        if [[ $new_gen -le $prev_gen ]]; then
+            stop_all_nodes
+            failexit "Generation did not increment after restart (prev=$prev_gen, new=$new_gen)"
+        else
+            echo "  SUCCESS: Generation incremented ($prev_gen -> $new_gen)"
+        fi
+
+        prev_gen=$new_gen
+    done
+
+    final_gen=$(get_node_gen $master)
+    echo "=== Test completed successfully ==="
+    echo "Summary:"
+    echo "  Initial generation: $initial_gen"
+    echo "  Final generation: $final_gen"
+    echo "  Total cluster restarts: 10"
+    echo "  Generation preserved and incremented correctly: YES"
+}
+
+run_test
+stop_all_nodes
+echo "Success"

tests/sqllogfill_reset_gen.test/Makefile

@@ -0,0 +1,10 @@
+ifeq ($(TESTSROOTDIR),)
+  include ../testcase.mk
+else
+  include $(TESTSROOTDIR)/testcase.mk
+endif
+export CHECK_DB_AT_FINISH=0
+ifeq ($(TEST_TIMEOUT),)
+	export TEST_TIMEOUT=10m
+endif
+

tests/sqllogfill_reset_gen.test/lrl.options

@@ -0,0 +1,11 @@
+# Disable features that prevent generation from going backwards
+berkattr elect_highest_committed_gen 0
+retrieve_gen_from_ckp 0
+match_on_ckp 0
+emit_gen_commits 0
+endianize_locklist 0
+sql_logfill 1
+sql_logfill_debug 1
+recovery_ckp 0
+
+logmsg level debug