When mostly idle, periodically ask for missed records

Signed-off-by: Michael Ponomarenko <mponomarenko@bloomberg.net>

Author: mponomar

bdb/rep.c

@@ -5564,8 +5564,9 @@ void *watcher_thread(void *arg)
             int nrecs;
             // if we're not seeing records or seeing only very old records, poke replication
             // to request records in the range we expect
-            if (gbl_nudge_replication_when_idle && bdb_state->dbenv->get_rep_lsns(bdb_state->dbenv, &next_lsn, &gap_lsn, &nrecs) == 0) {
-                if (nrecs == 0 && !IS_ZERO_LSN(gap_lsn)) {
+            if (gbl_nudge_replication_when_idle > 0 &&
+                bdb_state->dbenv->get_rep_lsns(bdb_state->dbenv, &next_lsn, &gap_lsn, &nrecs) == 0) {
+                if (nrecs < gbl_nudge_replication_when_idle && !IS_ZERO_LSN(gap_lsn)) {
                     DB_LSN tmp_lsn = {0};
                     DBT max_lsn_dbt = {0};
                     LOGCOPY_TOLSN(&tmp_lsn, &gap_lsn);

db/db_tunables.c

@@ -599,7 +599,7 @@ int64_t gbl_test_tunable_int64_limit = INT64_MAX;
 int64_t gbl_test_tunable_int64_signed_limit = INT64_MAX;
 
 int gbl_always_request_log_req = 0;
-int gbl_nudge_replication_when_idle = 0;
+int gbl_nudge_replication_when_idle = 100;
 
 int parse_int64(const char *value, int64_t *num);
 

db/db_tunables.h

@@ -2561,5 +2561,7 @@ REGISTER_TUNABLE("iam_dbname",
                  NULL, NULL, NULL);
 REGISTER_TUNABLE("queue_nonodh_scan_limit", "For comdb2_queues, stop queue scan at this depth (Default: 10000)", TUNABLE_INTEGER, &gbl_nonodh_queue_scan_limit, 0, NULL, NULL, NULL, NULL);
 REGISTER_TUNABLE("always_request_log_req", "Always request the next log record on replicant if there is a gap (default: off)", TUNABLE_BOOLEAN, &gbl_always_request_log_req, 0, NULL, NULL, NULL, NULL);
-REGISTER_TUNABLE("nudge_replication_when_idle", "If we haven't seen any replication events in a while, request some (default: off)", TUNABLE_BOOLEAN, &gbl_nudge_replication_when_idle, 0, NULL, NULL, NULL, NULL);
+REGISTER_TUNABLE("nudge_replication_when_idle",
+                 "If we haven't seen any replication events in a while, request some (default: 100)", TUNABLE_INTEGER,
+                 &gbl_nudge_replication_when_idle, 0, NULL, NULL, NULL, NULL);
 #endif /* _DB_TUNABLES_H */

tests/catchup_idle.test/Makefile

@@ -0,0 +1,8 @@
+ifeq ($(TESTSROOTDIR),)
+  include ../testcase.mk
+else
+  include $(TESTSROOTDIR)/testcase.mk
+endif
+ifeq ($(TEST_TIMEOUT),)
+  export TEST_TIMEOUT=60m
+endif

tests/catchup_idle.test/runit

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+bash -n "$0" | exit 1
+
+dbnm=$1
+
+if [ "x$dbnm" == "x" ] ; then
+    failexit "need a DB name"
+fi
+
+source ${TESTSROOTDIR}/tools/runit_common.sh
+source ${TESTSROOTDIR}/tools/waitmach.sh
+
+function wait_for_coherent_state() {
+    local host=$1
+    local desired_state=$2
+    local master=$3
+    local state=""
+    while :; do
+        state=$(cdb2sql --tabs --host ${master} ${CDB2_OPTIONS} ${DBNAME} default "select coherent_state from comdb2_cluster where host='$host'")
+        if [[ "$state" == "$desired_state" ]]; then
+            break
+        fi
+        echo "Waiting for $host to become $desired_state (currently $state)"
+        sleep 1
+    done
+}
+
+
+
+$CDB2SQL_EXE ${CDB2_OPTIONS} ${DBNAME} default "create table if not exists t(a int, b blob)"
+$CDB2SQL_EXE ${CDB2_OPTIONS} ${DBNAME} default "truncate t"
+
+echo "Populating"
+for i in $(seq 1 3000); do
+    $CDB2SQL_EXE ${CDB2_OPTIONS} ${DBNAME} default "insert into t values($i, randomblob(700000))"
+done
+
+host=$($CDB2SQL_EXE --tabs ${CDB2_OPTIONS} ${DBNAME} default "select host from comdb2_cluster where is_master='N' and coherent_state='coherent' limit 1")
+delhost=$($CDB2SQL_EXE --tabs ${CDB2_OPTIONS} ${DBNAME} default "select host from comdb2_cluster where is_master='N' and coherent_state='coherent' and host != '$host' limit 1")
+master=$($CDB2SQL_EXE --tabs ${CDB2_OPTIONS} ${DBNAME} default "select host from comdb2_cluster where is_master='Y'")
+
+echo "Using node $host"
+ssh $host "${CDB2SQL_EXE} -admin ${CDB2_OPTIONS} ${DBNAME} @localhost 'put tunable rep_debug_delay 10'"
+$CDB2SQL_EXE --host $delhost ${CDB2_OPTIONS} ${DBNAME} default "delete from t limit 3000" &
+sleep 10
+ssh $host "${CDB2SQL_EXE} -admin  ${CDB2_OPTIONS} ${DBNAME} @localhost 'put tunable rep_debug_delay 0'"
+
+# we expect the node we artificially delayed to become incoherent first, then coherent
+wait_for_coherent_state $host "INCOHERENT" $master
+wait_for_coherent_state $host "coherent" $master
+
+# wait for the node to actually become available (which may happen after it becomes coherent)
+# if we don't do this, then the test will fail with 'db unavailable at finish'.
+waitmach $host
+
+echo "Success"