SERVER-42476 Improve free monitoring tests

Author: markbenvenuto

jstests/free_mon/free_mon_http_down.js

@@ -21,8 +21,10 @@ const admin = conn.getDB('admin');
 
 mock_web.waitRegisters(3);
 
-const freeMonStats = assert.commandWorked(admin.runCommand({serverStatus: 1})).freeMonitoring;
-assert.gte(freeMonStats.registerErrors, 3);
+assert.soon(function() {
+    const freeMonStats = FreeMonGetServerStatus(conn);
+    return freeMonStats.registerErrors >= 3;
+}, "Failed to wait for 3 register errors: " + FreeMonGetServerStatus(conn), 20 * 1000);
 
 MongoRunner.stopMongod(conn);
 

jstests/free_mon/free_mon_metrics_halt.js

@@ -21,9 +21,7 @@ assert.neq(null, conn, 'mongod was unable to start up');
 mock_web.waitMetrics(6);
 
 // It gets marked as disabled on halt
-const reg = FreeMonGetRegistration(conn);
-print(tojson(reg));
-assert.eq(reg.state, "disabled");
+WaitForUnRegistration(conn);
 
 MongoRunner.stopMongod(conn);
 

jstests/free_mon/free_mon_metrics_perm_del.js

@@ -21,9 +21,13 @@ assert.neq(null, conn, 'mongod was unable to start up');
 mock_web.waitMetrics(4);
 
 // Make sure the registration document gets removed
-const reg = FreeMonGetRegistration(conn);
-print(tojson(reg));
-assert.eq(reg, undefined);
+assert.soon(
+    function() {
+        const reg = FreeMonGetRegistration(conn);
+        return reg === undefined;
+    },
+    "Failed to wait for free mon document to be removed: " + FreeMonGetRegistration(conn),
+    20 * 1000);
 
 MongoRunner.stopMongod(conn);
 

jstests/free_mon/free_mon_register_cmd.js

@@ -28,6 +28,8 @@ assert.eq(0, mock_web.queryStats().registers, "mongod registered without enablin
 
 assert.commandWorked(conn.adminCommand({setFreeMonitoring: 1, action: "enable"}));
 
+WaitForFreeMonServerStatusState(conn, 'enabled');
+
 // The command should either timeout or suceed after registration is complete
 const retStatus1 = conn.adminCommand({getFreeMonitoringStatus: 1});
 assert.commandWorked(retStatus1);

jstests/free_mon/free_mon_rs_delete.js

@@ -29,21 +29,26 @@ WaitForRegistration(rst.getSecondary());
 
 const qs1 = mock_web.queryStats();
 
+jsTestLog("Breaking Free Monitoring");
+
 // For kicks, delete the free monitoring storage state to knock free mon offline
 // and make sure the node does not crash
 rst.getPrimary().getDB("admin").system.version.remove({_id: "free_monitoring"});
 
+jsTestLog("Sleeping for 20s");
 sleep(20 * 1000);
 
 const qs2 = mock_web.queryStats();
 
+// Make sure we are back to the initial state.
+WaitForFreeMonServerStatusState(rst.getPrimary(), 'undecided');
+WaitForFreeMonServerStatusState(rst.getSecondary(), 'undecided');
+
 // Verify free monitoring stops but tolerate one additional collection
 assert.gte(qs1.metrics + 2, qs2.metrics);
-assert.eq(qs1.registers, qs2.registers);
-
-// Make sure we are back to the initial state.
-assert.eq(FreeMonGetServerStatus(rst.getPrimary()).state, 'undecided');
-assert.eq(FreeMonGetServerStatus(rst.getSecondary()).state, 'undecided');
+// Tolerate an additional registration on the secondary side. We may delete the record on the
+// primary before the secondary processes all pending registrations requests.
+assert.lte(qs1.registers, qs2.registers);
 
 // Enable it again to be sure we can resume
 assert.commandWorked(rst.getPrimary().adminCommand({setFreeMonitoring: 1, action: "enable"}));

jstests/free_mon/free_mon_rs_halt.js

@@ -23,8 +23,8 @@ WaitForRegistration(rst.getPrimary());
 
 mock_web.waitRegisters(2);
 
-assert.eq(FreeMonGetServerStatus(rst.getPrimary()).state, 'enabled');
-assert.eq(FreeMonGetServerStatus(rst.getSecondary()).state, 'enabled');
+WaitForFreeMonServerStatusState(rst.getPrimary(), 'enabled');
+WaitForFreeMonServerStatusState(rst.getSecondary(), 'enabled');
 
 mock_web.enableFaults();
 mock_web.waitFaults(1);
@@ -40,8 +40,8 @@ assert.gte(qs1.metrics + 1, qs2.metrics);
 assert.eq(qs1.registers, qs2.registers);
 
 // Halt causes us to disable free monitoring, not return it to initial state.
-assert.eq(FreeMonGetServerStatus(rst.getPrimary()).state, 'disabled');
-assert.eq(FreeMonGetServerStatus(rst.getSecondary()).state, 'disabled');
+WaitForFreeMonServerStatusState(rst.getPrimary(), 'disabled');
+WaitForFreeMonServerStatusState(rst.getSecondary(), 'disabled');
 
 // Disable the fault so we can re-enable again
 mock_web.disableFaults();
@@ -53,8 +53,8 @@ WaitForRegistration(rst.getSecondary());
 
 sleep(20 * 1000);
 
-assert.eq(FreeMonGetServerStatus(rst.getPrimary()).state, 'enabled');
-assert.eq(FreeMonGetServerStatus(rst.getSecondary()).state, 'enabled');
+WaitForFreeMonServerStatusState(rst.getPrimary(), 'enabled');
+WaitForFreeMonServerStatusState(rst.getSecondary(), 'enabled');
 
 rst.stopSet();
 

jstests/free_mon/free_mon_rs_perm_del.js

@@ -33,9 +33,8 @@ mock_web.waitFaults(1);
 sleep(20 * 1000);
 
 // Make sure we are back to the initial state.
-assert.eq(FreeMonGetServerStatus(rst.getPrimary()).state, 'undecided');
-
-assert.eq(FreeMonGetServerStatus(rst.getSecondary()).state, 'undecided');
+WaitForFreeMonServerStatusState(rst.getPrimary(), 'undecided');
+WaitForFreeMonServerStatusState(rst.getSecondary(), 'undecided');
 
 // Disable the fault so we can re-enable again
 mock_web.disableFaults();
@@ -47,8 +46,8 @@ WaitForRegistration(rst.getSecondary());
 
 sleep(20 * 1000);
 
-assert.eq(FreeMonGetServerStatus(rst.getPrimary()).state, 'enabled');
-assert.eq(FreeMonGetServerStatus(rst.getSecondary()).state, 'enabled');
+WaitForFreeMonServerStatusState(rst.getPrimary(), 'enabled');
+WaitForFreeMonServerStatusState(rst.getSecondary(), 'enabled');
 
 rst.stopSet();
 

jstests/free_mon/free_mon_rs_register.js

@@ -72,8 +72,8 @@ assert.commandWorked(rst.getPrimary().adminCommand({setFreeMonitoring: 1, action
 WaitForUnRegistration(rst.getPrimary());
 WaitForUnRegistration(rst.getSecondary());
 
-assert.eq(FreeMonGetServerStatus(rst.getPrimary()).state, 'disabled');
-assert.eq(FreeMonGetServerStatus(rst.getSecondary()).state, 'disabled');
+WaitForFreeMonServerStatusState(rst.getPrimary(), 'disabled');
+WaitForFreeMonServerStatusState(rst.getSecondary(), 'disabled');
 
 // Restart the secondary with it disabled
 var s1 = rst._slaves[0];
@@ -85,8 +85,8 @@ rst.waitForState(s1, ReplSetTest.State.DOWN);
 rst.restart(s1Id);
 
 // Make sure it is disabled
-assert.eq(FreeMonGetServerStatus(rst.getPrimary()).state, 'disabled');
-assert.eq(FreeMonGetServerStatus(rst.getSecondary()).state, 'disabled');
+WaitForFreeMonServerStatusState(rst.getPrimary(), 'disabled');
+WaitForFreeMonServerStatusState(rst.getSecondary(), 'disabled');
 
 rst.stopSet();
 

jstests/free_mon/free_mon_rs_resend.js

@@ -39,8 +39,8 @@ WaitForRegistration(rst.getPrimary());
 mock_web.waitRegisters(1);
 mock_web_sec.waitRegisters(1);
 
-assert.eq(FreeMonGetServerStatus(rst.getPrimary()).state, 'enabled');
-assert.eq(FreeMonGetServerStatus(rst.getSecondary()).state, 'enabled');
+WaitForFreeMonServerStatusState(rst.getPrimary(), 'enabled');
+WaitForFreeMonServerStatusState(rst.getSecondary(), 'enabled');
 
 mock_web.waitRegisters(2);
 mock_web_sec.waitRegisters(2);

jstests/free_mon/free_mon_server_status.js

@@ -34,7 +34,8 @@ assert.eq(enabled.metricsErrors, 0);
 
 // Explicitly disabled.
 admin.disableFreeMonitoring();
-sleep(2);  // Give the async command time to run.
+
+WaitForFreeMonServerStatusState(mongod, "disabled");
 
 const disabled = freeMonStats();
 assert.eq(disabled.state, 'disabled');