Skip to content

Commit 9341353

Browse files
markhannummarkhannum
committed
Physrep masters are always coherent
Signed-off-by: Mark Hannum <mhannum@bloomberg.net>
1 parent 163e967 commit 9341353

File tree

4 files changed

+160
-4
lines changed

4 files changed

+160
-4
lines changed

‎bdb/rep.c‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
#include "printformats.h"
4646
#include "phys_rep.h"
4747
#include "phys_rep_lsn.h"
48+
#include "truncate_log.h"
4849
#include <compat.h>
4950
#include "str0.h"
5051
#include <thrman.h>
@@ -4317,10 +4318,21 @@ int bdb_am_i_coherent(bdb_state_type *bdb_state)
43174318
int bdb_try_am_i_coherent(bdb_state_type *bdb_state)
43184319
{
43194320
int coherent = 0;
4321+
int isphysrep = gbl_is_physical_replicant;
43204322

43214323
if (bdb_state->parent)
43224324
bdb_state = bdb_state->parent;
43234325

4326+
if (isphysrep) {
4327+
/* If this is a physrep and log truncation is in progress,
4328+
* we must be a standalone installation or a physrep master.
4329+
* These are always coherent. */
4330+
int rc = truncate_trylock();
4331+
if (rc != 0) {
4332+
return 1;
4333+
}
4334+
}
4335+
43244336
/*
43254337
* If we cannot get the bdb-lock, that means the lock is desired,
43264338
* which only happens during election. We are not coherent if the
@@ -4331,6 +4343,10 @@ int bdb_try_am_i_coherent(bdb_state_type *bdb_state)
43314343
BDB_RELLOCK();
43324344
}
43334345

4346+
if (isphysrep) {
4347+
truncate_unlock();
4348+
}
4349+
43344350
return coherent;
43354351
}
43364352

‎db/truncate_log.c‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,18 @@ extern struct dbenv *thedb;
1212
LOG_INFO find_match_lsn(void *bdb_state, cdb2_hndl_tp *repl_db,
1313
LOG_INFO start_info);
1414

15+
static pthread_mutex_t physrep_truncate_lock = PTHREAD_MUTEX_INITIALIZER;
16+
17+
int truncate_trylock(void)
18+
{
19+
return pthread_mutex_trylock(&physrep_truncate_lock);
20+
}
21+
22+
void truncate_unlock(void)
23+
{
24+
Pthread_mutex_unlock(&physrep_truncate_lock);
25+
}
26+
1527
LOG_INFO handle_truncation(cdb2_hndl_tp *repl_db, LOG_INFO latest_info)
1628
{
1729
LOG_INFO match_lsn = find_match_lsn(thedb->bdb_env, repl_db, latest_info);
@@ -27,7 +39,9 @@ LOG_INFO handle_truncation(cdb2_hndl_tp *repl_db, LOG_INFO latest_info)
2739
match_lsn.offset);
2840
}
2941

42+
Pthread_mutex_lock(&physrep_truncate_lock);
3043
truncate_log(match_lsn.file, match_lsn.offset, 1);
44+
Pthread_mutex_unlock(&physrep_truncate_lock);
3145

3246
return match_lsn;
3347
}

‎db/truncate_log.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77

88
int truncate_log(unsigned int file, unsigned int offset, uint32_t flags);
99
int truncate_timestamp(time_t timestamp);
10+
int truncate_trylock(void);
11+
void truncate_unlock(void);
1012
LOG_INFO handle_truncation(cdb2_hndl_tp *repl_db, LOG_INFO prev_info);
1113

1214
#endif

‎tests/phys_rep_tiered.test/runit‎

Lines changed: 128 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1264,6 +1264,128 @@ function verify_log_cursor_gen()
12641264
done
12651265
}
12661266

1267+
function select_from_master()
1268+
{
1269+
local dbname=$1
1270+
local node=$2
1271+
local timeout=$3
1272+
local end=$(( $(date +%s) + timeout ))
1273+
1274+
while [[ $(date +%s) -lt $end ]]; do
1275+
$CDB2SQL_EXE --tabs ${CDB2_OPTIONS} $dbname --host $node "select comdb2_host()" > /dev/null 2>&1
1276+
done
1277+
}
1278+
1279+
# Physrep masters and standalone nodes should never return 'incoherent'
1280+
function incoherent_master()
1281+
{
1282+
local physrep_master=""
1283+
local name=""
1284+
local node=""
1285+
local logfile=""
1286+
local cnt=""
1287+
local timeout=60
1288+
local foundbad=0
1289+
local beforecnt=0
1290+
local aftercnt=0
1291+
local pids=()
1292+
1293+
echo "== Verifying master/standalone does not return incoherent"
1294+
1295+
echo "Set default-timeout to 5 seconds on all nodes"
1296+
for node in $CLUSTER ; do
1297+
$CDB2SQL_EXE $CDB2_OPTIONS $DBNAME --host $node "put tunable tranlog_default_timeout 5"
1298+
name="${REPL_DBNAME_PREFIX}_${node}"
1299+
$CDB2SQL_EXE $CDB2_OPTIONS $name --host $node "put tunable tranlog_default_timeout 5"
1300+
$CDB2SQL_EXE $CDB2_OPTIONS $REPL_CLUS_DBNAME --host $node "put tunable tranlog_default_timeout 5"
1301+
done
1302+
1303+
$CDB2SQL_EXE $CDB2_OPTIONS $DBNAME default "create table xxx(a int)"
1304+
1305+
physrep_master=$($CDB2SQL_EXE --tabs ${CDB2_OPTIONS} $REPL_CLUS_DBNAME --host $firstNode "select host from comdb2_cluster where is_master='Y'")
1306+
1307+
# Verify that each physrep has this table
1308+
for node in $CLUSTER ; do
1309+
name="${REPL_DBNAME_PREFIX}_${node}"
1310+
cnt=$($CDB2SQL_EXE --tabs ${CDB2_OPTIONS} $name --host $node "select count(*) from comdb2_tables where tablename='xxx'")
1311+
while [[ "$cnt" -ne "1" ]]; do
1312+
sleep 1
1313+
cnt=$($CDB2SQL_EXE --tabs ${CDB2_OPTIONS} $name --host $node "select count(*) from comdb2_tables where tablename='xxx'")
1314+
done
1315+
1316+
cnt=$($CDB2SQL_EXE --tabs ${CDB2_OPTIONS} $REPL_CLUS_DBNAME --host $node "select count(*) from comdb2_tables where tablename='xxx'")
1317+
while [[ "$cnt" -ne "1" ]]; do
1318+
sleep 1
1319+
cnt=$($CDB2SQL_EXE --tabs ${CDB2_OPTIONS} $REPL_CLUS_DBNAME --host $node "select count(*) from comdb2_tables where tablename='xxx'")
1320+
done
1321+
done
1322+
1323+
echo "Getting before-count from current clustered physrep master"
1324+
logfile=$TESTDIR/logs/${REPL_CLUS_DBNAME}.${physrep_master}.log
1325+
beforecnt=$(egrep "new query on incoherent node" $logfile | wc -l)
1326+
1327+
echo "Spawning select-from-master threads"
1328+
select_from_master $REPL_CLUS_DBNAME $physrep_master $timeout &
1329+
pids+=($!)
1330+
1331+
for node in $CLUSTER ; do
1332+
name="${REPL_DBNAME_PREFIX}_${node}"
1333+
select_from_master $name $node $timeout &
1334+
pids+=($!)
1335+
done
1336+
1337+
echo "Waiting on all spawned pids"
1338+
for p in "${pids[@]}" ; do
1339+
wait $p
1340+
done
1341+
1342+
echo "Checking for incoherent trace in output files"
1343+
for node in $CLUSTER ; do
1344+
name="${REPL_DBNAME_PREFIX}_${node}"
1345+
logfile=$TESTDIR/logs/${name}.log
1346+
egrep "new query on incoherent node" $logfile >/dev/null 2>&1
1347+
if [[ $? -eq 0 ]]; then
1348+
echo "Found incoherent trace in $logfile"
1349+
foundbad=1
1350+
fi
1351+
done
1352+
1353+
logfile=$TESTDIR/logs/${REPL_CLUS_DBNAME}.${physrep_master}.log
1354+
aftercnt=$(egrep "new query on incoherent node" $logfile | wc -l)
1355+
if [[ "$beforecnt" -ne "$aftercnt" ]]; then
1356+
echo "Found additional incoherent in $logfile"
1357+
foundbad=1
1358+
fi
1359+
1360+
[[ "$foundbad" -eq 1 ]] && cleanFailExit "incoherent_master test failed"
1361+
1362+
echo "Drop table xxx"
1363+
$CDB2SQL_EXE ${CDB2_OPTIONS} $DBNAME default "drop table xxx"
1364+
1365+
# Verify that each physrep has dropped this table
1366+
for node in $CLUSTER ; do
1367+
name="${REPL_DBNAME_PREFIX}_${node}"
1368+
cnt=$($CDB2SQL_EXE --tabs ${CDB2_OPTIONS} $name --host $node "select count(*) from comdb2_tables where tablename='xxx'")
1369+
while [[ "$cnt" -ne "0" ]]; do
1370+
sleep 1
1371+
cnt=$($CDB2SQL_EXE --tabs ${CDB2_OPTIONS} $name --host $node "select count(*) from comdb2_tables where tablename='xxx'")
1372+
done
1373+
cnt=$($CDB2SQL_EXE --tabs ${CDB2_OPTIONS} $REPL_CLUS_DBNAME --host $node "select count(*) from comdb2_tables where tablename='xxx'")
1374+
while [[ "$cnt" -ne "0" ]]; do
1375+
sleep 1
1376+
cnt=$($CDB2SQL_EXE --tabs ${CDB2_OPTIONS} $REPL_CLUS_DBNAME --host $node "select count(*) from comdb2_tables where tablename='xxx'")
1377+
done
1378+
done
1379+
1380+
echo "Set default-timeout back to 30 seconds on all nodes"
1381+
for node in $CLUSTER ; do
1382+
$CDB2SQL_EXE $CDB2_OPTIONS $DBNAME --host $node "put tunable tranlog_default_timeout 30"
1383+
name="${REPL_DBNAME_PREFIX}_${node}"
1384+
$CDB2SQL_EXE $CDB2_OPTIONS $name --host $node "put tunable tranlog_default_timeout 30"
1385+
$CDB2SQL_EXE $CDB2_OPTIONS $REPL_CLUS_DBNAME --host $node "put tunable tranlog_default_timeout 30"
1386+
done
1387+
}
1388+
12671389
function revconn_latency()
12681390
{
12691391
typeset now=$(date +%s)
@@ -2241,8 +2363,6 @@ function verify_non_ignored_reptype
22412363
[[ -n "$ignored" ]] && cleanFailExit "Found ignored rep-messages after incoherent recovery: $ignored"
22422364
}
22432365

2244-
# egrep "Ignoring rp->gen" * | egrep "rectype=2\>|rectype=11\>|rectype=10\>"
2245-
22462366
function announce
22472367
{
22482368
typeset text=$1
@@ -2402,6 +2522,11 @@ function run_tests
24022522
verify_alt_metadb_system_table
24032523
testcase_finish $testcase
24042524

2525+
testcase="incoherent_master"
2526+
testcase_preamble $testcase
2527+
incoherent_master
2528+
testcase_finish $testcase
2529+
24052530
testcase="physrep_allowed_source"
24062531
testcase_preamble $testcase
24072532
physrep_allowed_source
@@ -2410,10 +2535,9 @@ function run_tests
24102535

24112536
function run_one_test
24122537
{
2413-
physrep_allowed_source
2538+
incoherent_master
24142539
}
24152540

2416-
#run_one_test
24172541
run_tests
24182542
cleanup
24192543

0 commit comments

Comments
 (0)