Reduce physrep-sql against the metadb

markhannum · markhannum · commit f3ad4daca188 · 2026-03-22T18:13:22.000-04:00
Signed-off-by: Mark Hannum &lt;mhannum@bloomberg.net&gt;
diff --git a/db/db_tunables.c b/db/db_tunables.c
@@ -522,6 +522,7 @@ extern int gbl_physrep_update_registry_interval;
 extern int gbl_physrep_i_am_metadb;
 extern int gbl_physrep_keepalive_v2;
 extern int gbl_physrep_keepalive_freq_sec;
+extern int gbl_physrep_slow_replicant_check_freq_sec;
 extern int gbl_physrep_max_candidates;
 extern int gbl_physrep_reconnect_penalty;
 extern int gbl_physrep_reconnect_interval;
diff --git a/db/db_tunables.h b/db/db_tunables.h
@@ -1821,7 +1821,7 @@ REGISTER_TUNABLE("tranlog_incoherent_timeout", "Timeout in seconds for incoheren
                  TUNABLE_INTEGER, &gbl_tranlog_incoherent_timeout, 0, NULL, NULL, NULL, NULL);
 REGISTER_TUNABLE("tranlog_maxpoll", "Tranlog timeout in seconds for blocking poll. (Default: 60)", TUNABLE_INTEGER,
                  &gbl_tranlog_maxpoll, 0, NULL, NULL, NULL, NULL);
-REGISTER_TUNABLE("physrep_check_minlog_freq_sec", "Check the minimum log number to keep this often. (Default: 10)",
+REGISTER_TUNABLE("physrep_check_minlog_freq_sec", "Check the minimum log number to keep this often. (Default: 60)",
                  TUNABLE_INTEGER, &gbl_physrep_check_minlog_freq_sec, 0, NULL, NULL, NULL, NULL);
 REGISTER_TUNABLE("physrep_debug", "Print extended physrep trace. (Default: off)", TUNABLE_BOOLEAN, &gbl_physrep_debug,
                  0, NULL, NULL, NULL, NULL);
@@ -1830,10 +1830,8 @@ REGISTER_TUNABLE("physrep_exit_on_invalid_logstream", "Exit physreps on invalid
 REGISTER_TUNABLE("physrep_fanout",
                  "Maximum number of physical replicants that a node can service (Default: 8)",
                  TUNABLE_INTEGER, &gbl_physrep_fanout, 0, NULL, NULL, NULL, NULL);
-REGISTER_TUNABLE("physrep_hung_replicant_check_freq_sec",
-                 "Check for hung physical replicant this often. (Default: 10)",
-                 TUNABLE_INTEGER, &gbl_physrep_hung_replicant_check_freq_sec, 0, NULL,
-                 NULL, NULL, NULL);
+REGISTER_TUNABLE("physrep_hung_replicant_check_freq_sec", "Check for hung physical replicant this often. (Default: 60)",
+                 TUNABLE_INTEGER, &gbl_physrep_hung_replicant_check_freq_sec, 0, NULL, NULL, NULL, NULL);
 REGISTER_TUNABLE("physrep_hung_replicant_threshold",
                  "Report if the physical replicant has been inactive for this duration. (Default: 60)",
                  TUNABLE_INTEGER, &gbl_physrep_hung_replicant_threshold, 0, NULL,
@@ -1856,13 +1854,14 @@ REGISTER_TUNABLE("physrep_i_am_metadb", "I am physical replication metadb (Defau
 REGISTER_TUNABLE("physrep_keepalive_v2", "Use version 2 of keepalive which includes first lsn. (Default: off)",
                  TUNABLE_BOOLEAN, &gbl_physrep_keepalive_v2, 0, NULL, NULL, NULL, NULL);
 REGISTER_TUNABLE("physrep_keepalive_freq_sec",
-                 "Periodically send lsn to source node after this interval. (Default: 10)", TUNABLE_INTEGER,
+                 "Periodically send lsn to source node after this interval. (Default: 60)", TUNABLE_INTEGER,
                  &gbl_physrep_keepalive_freq_sec, 0, NULL, NULL, NULL, NULL);
+REGISTER_TUNABLE("physrep_slow_replicant_check_freq_sec", "Check for slow physical replicant this often. (Default: 60)",
+                 TUNABLE_INTEGER, &gbl_physrep_slow_replicant_check_freq_sec, 0, NULL, NULL, NULL, NULL);
 REGISTER_TUNABLE("physrep_max_candidates",
                  "Maximum number of candidates that should be returned to a "
                  "new physical replicant during registration. (Default: 6)",
-                 TUNABLE_INTEGER, &gbl_physrep_max_candidates, 0, NULL,
-                 NULL, NULL, NULL);
+                 TUNABLE_INTEGER, &gbl_physrep_max_candidates, 0, NULL, NULL, NULL, NULL);
 REGISTER_TUNABLE("physrep_metadb_host", "List of physical replication metadb cluster hosts.", TUNABLE_STRING,
                  &gbl_physrep_metadb_host, READONLY, NULL, NULL, NULL, NULL);
 REGISTER_TUNABLE("physrep_metadb_name", "Physical replication metadb cluster name.",
@@ -1923,7 +1922,7 @@ REGISTER_TUNABLE("revsql_debug",
                  TUNABLE_BOOLEAN, &gbl_revsql_debug, EXPERIMENTAL | INTERNAL,
                  NULL, NULL, NULL, NULL);
 REGISTER_TUNABLE("revsql_host_refresh_freq_sec", "The frequency at which the "
-                 "reverse connection host list will be refreshed (Default: 5secs)",
+                 "reverse connection host list will be refreshed (Default: 60 secs)",
                  TUNABLE_INTEGER, &gbl_revsql_host_refresh_freq_sec, EXPERIMENTAL | INTERNAL,
                  NULL, NULL, NULL, NULL);
 REGISTER_TUNABLE("admin_revsql", "Run revsql sessions as admin.  (Default: Off)", TUNABLE_BOOLEAN, &gbl_admin_revsql, 0,
diff --git a/db/phys_rep.c b/db/phys_rep.c
@@ -67,11 +67,10 @@ int gbl_physrep_fanout = 8;
 int gbl_physrep_max_candidates = 6;
 int gbl_physrep_max_pending_replicants = 10;
 int gbl_deferred_phys_flag = 0;
-int gbl_physrep_source_nodes_refresh_freq_sec = 10;
-int gbl_physrep_slow_replicant_check_freq_sec = 10;
-int gbl_physrep_keepalive_freq_sec = 10;
-int gbl_physrep_check_minlog_freq_sec = 10;
-int gbl_physrep_hung_replicant_check_freq_sec = 10;
+int gbl_physrep_slow_replicant_check_freq_sec = 60;
+int gbl_physrep_keepalive_freq_sec = 60;
+int gbl_physrep_hung_replicant_check_freq_sec = 60;
+int gbl_physrep_check_minlog_freq_sec = 600;
 int gbl_physrep_hung_replicant_threshold = 60;
 int gbl_physrep_revconn_check_interval = 60;
 int gbl_physrep_update_registry_interval = 60;
@@ -1107,27 +1106,10 @@ int gbl_physrep_keepalive_v2 = 0;
 
 static int send_keepalive_int(cdb2_hndl_tp *metadb)
 {
-    int rc = 0, use_v2 = 0;
+    int rc = 0, use_v2 = gbl_physrep_keepalive_v2;
     char cmd[600];
     LOG_INFO info;
 
-    /* TODO: remove after v2 enabled & new-schema is everywhere */
-    if (gbl_physrep_keepalive_v2) {
-        rc = snprintf(
-            cmd, sizeof(cmd),
-            "select count(*) from comdb2_columns where tablename='comdb2_physreps' and columnname='firstfile'");
-        ATOMIC_ADD64(gbl_physrep_metadb_sql_count, 1);
-        rc = cdb2_run_statement(metadb, cmd);
-        if (rc != CDB2_OK) {
-            physrep_logmsg(LOGMSG_ERROR, "%s:%d Failed to execute cmd %s (rc: %d)\n", __func__, __LINE__, cmd, rc);
-            return rc;
-        }
-        if (cdb2_next_record(metadb) == CDB2_OK) {
-            int64_t val = *(int64_t *)cdb2_column_value(metadb, 0);
-            use_v2 = (val != 0) ? 1 : 0;
-        }
-    }
-
     info = get_last_lsn(thedb->bdb_env);
     if (use_v2) {
         LOG_INFO first_info;
@@ -1573,14 +1555,15 @@ static void *physrep_worker(void *args)
                     physrep_logmsg(LOGMSG_USER, "%s:%d Reverse connection check: do-revcon=%d, is-revcon=%d\n",
                                    __func__, __LINE__, do_revconn, is_revconn);
                 }
+
+                /* The call might have failed.  That's okay, don't hammer metadb */
+                last_revconn_check = comdb2_time_epoch();
+
                 if (do_revconn == -1) {
-                    logmsg(LOGMSG_ERROR, "%s:%d Failed to contact physrep metadb- keeping do_revconn the same: %d\n",
+                    logmsg(LOGMSG_DEBUG, "%s:%d Failed to contact physrep metadb- keeping do_revconn the same: %d\n",
                            __func__, __LINE__, is_revconn);
                 } else {
 
-                    /* Only update timestamp on successful check */
-                    last_revconn_check = comdb2_time_epoch();
-
                     if ((do_revconn && !is_revconn) || (!do_revconn && is_revconn)) {
                         logmsg(LOGMSG_USER, "Revconn changed, do_revconn=%d, is_revconn=%d\n", do_revconn, is_revconn);
                         close_repl_connection(repl_db_cnct, repl_db, __func__, __LINE__);
diff --git a/db/reverse_conn.c b/db/reverse_conn.c
@@ -68,7 +68,7 @@ int gbl_revsql_allow_command_exec;
 int gbl_revsql_debug = 0;
 int gbl_revsql_cdb2_debug;
 // 'reverse-connection host' list refresh frequency
-int gbl_revsql_host_refresh_freq_sec = 5;
+int gbl_revsql_host_refresh_freq_sec = 60;
 // 'reverse-connection' worker's new connection attempt frequency
 int gbl_revsql_connect_freq_sec = 5;
 
diff --git a/docs/pages/operating/physical_replication.md b/docs/pages/operating/physical_replication.md
@@ -171,7 +171,7 @@ CREATE TABLE comdb2_physrep_sources(dbname CSTRING(60),
 ## Tunables
 
 * blocking_physrep: The `SELECT .. FROM comdb2_transaction_logs` query executed by physical replicants blocks for the next log record. (Default: `false`)
-* physrep_check_minlog_freq_sec: Check the minimum log number to keep this often. (Default: `10`)
+* physrep_check_minlog_freq_sec: Check the minimum log number to keep this often. (Default: `60`)
 * physrep_debug: Print extended physrep trace. (Default: `off`)
 * physrep_exit_on_invalid_logstream: Exit physreps on invalid logstream. (Default: off)
 * physrep_fanout: Maximum number of physical replicants that a node can service (Default: `8`)
@@ -189,9 +189,9 @@ CREATE TABLE comdb2_physrep_sources(dbname CSTRING(60),
 * physrep_source_host: List of physical replication source cluster hosts.
 * revsql_allow_command_execution : Allow processing and execution of command * over the `reverse connection` that has come in as part of the request. This is mostly intended for testing. (Default: off)
 * revsql_cdb2_debug: Print extended reversql-sql cdb2 related trace. (Default: off)
-* revsql_connect_freq_sec: This node will attempt to `reverse connect` to the remote host at this frequency. (Default: 5secs)
+* revsql_connect_freq_sec: This node will attempt to `reverse connect` to the remote host at this frequency. (Default: 5 secs)
 * revsql_debug: Print extended reversql-sql trace. (Default: off)
-* revsql_host_refresh_freq_sec: The frequency at which the reverse connection host list will be refreshed. (Default: 5secs)
+* revsql_host_refresh_freq_sec: The frequency at which the reverse connection host list will be refreshed. (Default: 60 secs)
 * revsql_force_rte: Force rte-mode for all reverse connections. (Default: on)
 * connect_remote_rte: Force rte-mode for both fdb and revsql connections. (Default: off)
 
diff --git a/lua/lib/physrep_get_reverse_hosts.lua b/lua/lib/physrep_get_reverse_hosts.lua
@@ -5,14 +5,6 @@
 local function main(dbname, hostname)
     db:begin()
 
-    -- Check whether 'comdb2_physrep_sources' table exists
-    local rs, rc = db:exec("SELECT count(*)=1 AS cnt FROM comdb2_tables WHERE tablename = 'comdb2_physrep_sources'")
-    local row = rs:fetch()
-    if row.cnt == 0 then
-        db:commit()
-        return
-    end
-
     local rs, rc = db:exec("SELECT dbname, host FROM comdb2_physrep_sources WHERE " ..
                            "source_dbname = '" ..  dbname .. "' AND source_host = '" .. hostname .. "'")
     local row = rs:fetch()
diff --git a/lua/lib/physrep_get_reverse_hosts_v2.lua b/lua/lib/physrep_get_reverse_hosts_v2.lua
@@ -5,14 +5,6 @@
 local function main(dbname, hostname)
     db:begin()
 
-    -- Check whether 'comdb2_physrep_sources' table exists
-    local rs, rc = db:exec("SELECT count(*)=1 AS cnt FROM comdb2_tables WHERE tablename = 'comdb2_physrep_sources'")
-    local row = rs:fetch()
-    if row.cnt == 0 then
-        db:commit()
-        return
-    end
-
     local rs, rc = db:exec("SELECT dbname, host FROM comdb2_physrep_sources WHERE " ..
                            "source_dbname = '" ..  dbname .. "' AND source_host = '" .. hostname .. "'")
     local row = rs:fetch()
diff --git a/lua/lib/physrep_get_revhosts_v2.lua b/lua/lib/physrep_get_revhosts_v2.lua
@@ -5,14 +5,6 @@
 local function main(dbname, hostname, tier, cluster)
     db:begin()
 
-    -- Check whether 'comdb2_physrep_sources' table exists
-    local rs, rc = db:exec("SELECT count(*)=1 AS cnt FROM comdb2_tables WHERE tablename = 'comdb2_physrep_sources'")
-    local row = rs:fetch()
-    if row.cnt == 0 then
-        db:commit()
-        return
-    end
-
     local sql = ("SELECT dbname, host FROM comdb2_physrep_sources WHERE " ..
                  "source_dbname = '" ..  dbname .. "' AND ( source_host = '" .. hostname .. 
                  "' OR source_host = '" .. tier ..
diff --git a/lua/lib/physrep_should_wait_for_con.lua b/lua/lib/physrep_should_wait_for_con.lua
@@ -10,15 +10,6 @@
 local function main(dbname, hostname)
     db:begin()
 
-    local rs, row = db:exec("SELECT count(*)=1 AS cnt FROM comdb2_tables WHERE tablename = 'comdb2_physrep_sources'")
-    local row = rs:fetch()
-
-    if row.cnt == 0 then
-        db:emit(row)
-        db:commit()
-        return
-    end
-
     local rs, row = db:exec("SELECT count(*) as cnt FROM comdb2_physrep_sources " ..
                             "    WHERE dbname = '" ..  dbname .. "' AND " ..
                             "        host LIKE '" .. hostname .. "'")
diff --git a/lua/lib/physrep_shouldwait_v2.lua b/lua/lib/physrep_shouldwait_v2.lua
@@ -10,15 +10,6 @@
 local function main(dbname, hostname, tier, cluster)
     db:begin()
 
-    local rs, row = db:exec("SELECT count(*)=1 AS cnt FROM comdb2_tables WHERE tablename = 'comdb2_physrep_sources'")
-    local row = rs:fetch()
-
-    if row.cnt == 0 then
-        db:emit(row)
-        db:commit()
-        return
-    end
-
     local sql = ("SELECT count(*) as cnt from comdb2_physrep_sources " ..
                  "    WHERE dbname = '" .. dbname .. "' AND " ..
                  "        ( host LIKE '" .. hostname .. "' OR " ..
diff --git a/tests/phys_rep_tiered.test/firstfile.testopts b/tests/phys_rep_tiered.test/firstfile.testopts
@@ -0,0 +1 @@
+physrep_keepalive_v2 1
diff --git a/tests/phys_rep_tiered.test/lrl.options b/tests/phys_rep_tiered.test/lrl.options
@@ -2,6 +2,8 @@ logmsg level debug
 dtastripe 1
 debug_drop_nth_rep_message 10000
 incoherent_slow_inactive_timeout 0
+# test requires fast revconn than 
+revsql_host_refresh_freq_sec 5
 revsql_fake_connect_failure 1
 revsql_debug 1
 physrep_debug 1
@@ -10,7 +12,7 @@ forbid_remote_admin 0
 ctrace_dbdir 1
 allow_lua_print 1
 reverse_hosts_v2 1
-physrep_keepalive_v2 1
+physrep_keepalive_v2 0
 enable_snapshot_isolation
 
 # Reproduced 'incoherent-physrep' bug with this removed.
diff --git a/tests/phys_rep_tiered.test/runit b/tests/phys_rep_tiered.test/runit
@@ -2701,6 +2701,12 @@ function physrep_metadb_sql_count_test
     echo "TOTAL physrep_metadb_sql_count: $total_count"
     echo "========================================="
     echo ""
+
+    if [[ $total_count -gt 2500 ]]; then
+        cleanFailExit "FAILED: Total metadb SQL count ($total_count) exceeds threshold (2500)"
+    fi
+
+    echo "SUCCESS: Total metadb SQL count ($total_count) is within acceptable range (<= 2500)"
 }
 
 function announce
diff --git a/tests/physrep_ignore_table.test/lrl.options b/tests/physrep_ignore_table.test/lrl.options
@@ -1 +1,2 @@
 ufid_log on
+setattr MIN_KEEP_LOGS 20
diff --git a/tests/physrep_ignore_table.test/runit b/tests/physrep_ignore_table.test/runit
@@ -95,6 +95,7 @@ function create_physrep
 
     # ignore tables t2 and t4
     echo "physrep_ignore t2 t4" >> $DEST_DBDIR/${DESTDB}.lrl
+    echo "physrep_debug 1" >> $DEST_DBDIR/${DESTDB}.lrl
     
     export replog=$DEST_DBDIR/$DESTDB.log
     
diff --git a/tests/tunables.test/t00_all_tunables.expected b/tests/tunables.test/t00_all_tunables.expected
@@ -733,16 +733,16 @@
 (name='pgcompactpool.stacksz', description='Thread stack size.', type='INTEGER', value='1048576', read_only='N')
 (name='physical_ack_interval', description='For logical transactions, have the slave send an 'ack' after this many physical operations.', type='INTEGER', value='0', read_only='N')
 (name='physical_commit_interval', description='Force a physical commit after this many physical operations.', type='INTEGER', value='512', read_only='N')
-(name='physrep_check_minlog_freq_sec', description='Check the minimum log number to keep this often. (Default: 10)', type='INTEGER', value='10', read_only='N')
+(name='physrep_check_minlog_freq_sec', description='Check the minimum log number to keep this often. (Default: 60)', type='INTEGER', value='60', read_only='N')
 (name='physrep_debug', description='Print extended physrep trace. (Default: off)', type='BOOLEAN', value='OFF', read_only='N')
 (name='physrep_exit_on_invalid_logstream', description='Exit physreps on invalid logstream.  (Default: off)', type='BOOLEAN', value='OFF', read_only='N')
 (name='physrep_fanout', description='Maximum number of physical replicants that a node can service (Default: 8)', type='INTEGER', value='8', read_only='N')
 (name='physrep_filter_by_class', description='Filter physrep replication by class. (Default: on)', type='BOOLEAN', value='ON', read_only='N')
-(name='physrep_hung_replicant_check_freq_sec', description='Check for hung physical replicant this often. (Default: 10)', type='INTEGER', value='10', read_only='N')
+(name='physrep_hung_replicant_check_freq_sec', description='Check for hung physical replicant this often. (Default: 60)', type='INTEGER', value='60', read_only='N')
 (name='physrep_hung_replicant_threshold', description='Report if the physical replicant has been inactive for this duration. (Default: 60)', type='INTEGER', value='60', read_only='N')
 (name='physrep_i_am_metadb', description='I am physical replication metadb (Default: off)', type='BOOLEAN', value='OFF', read_only='N')
 (name='physrep_ignore_queues', description='Don't replicate queues.', type='BOOLEAN', value='ON', read_only='Y')
-(name='physrep_keepalive_freq_sec', description='Periodically send lsn to source node after this interval. (Default: 10)', type='INTEGER', value='10', read_only='N')
+(name='physrep_keepalive_freq_sec', description='Periodically send lsn to source node after this interval. (Default: 60)', type='INTEGER', value='60', read_only='N')
 (name='physrep_keepalive_v2', description='Use version 2 of keepalive which includes first lsn. (Default: off)', type='BOOLEAN', value='OFF', read_only='N')
 (name='physrep_max_candidates', description='Maximum number of candidates that should be returned to a new physical replicant during registration. (Default: 6)', type='INTEGER', value='6', read_only='N')
 (name='physrep_max_rollback', description='Maximum logs physrep can rollback. (Default: 0)', type='INTEGER', value='0', read_only='N')
@@ -755,6 +755,7 @@
 (name='physrep_repl_name', description='Current physrep parent.', type='STRING', value=NULL, read_only='Y')
 (name='physrep_revconn_check_interval', description='Physrep recheck revconn interval.  (Default: 60)', type='INTEGER', value='60', read_only='N')
 (name='physrep_shuffle_host_list', description='Shuffle the host list returned by register_replicant() before connecting to the hosts. (Default: OFF)', type='BOOLEAN', value='OFF', read_only='N')
+(name='physrep_slow_replicant_check_freq_sec', description='Check for slow physical replicant this often. (Default: 60)', type='INTEGER', value='60', read_only='N')
 (name='physrep_source_dbname', description='Physical replication source cluster dbname.', type='STRING', value=NULL, read_only='Y')
 (name='physrep_source_dbnum', description='Physical replication source cluster db number.', type='INTEGER', value='0', read_only='Y')
 (name='physrep_source_host', description='List of physical replication source cluster hosts.', type='STRING', value=NULL, read_only='Y')
