Fix test container_checker monit flaky issue. (#21218)

These changes fix the sonic-mgmt test "container_checker.py" flaky issue by fixing
the following issues in pytest fixture "update_monit_service":
 - fixing bug updating monit conf "/etc/monit/conf.d/sonic-host" for
   container_checker periodicity.
   (Currently the sed commands are updating the last line rule, which does not
    correctly update the container_checker alert periodicity rule.)
 - Adds wait_until after monit restart, to ensure tests are running when monit
   is properly up.

related bug: aristanetworks/sonic-qual.msft#906

Author: prajjwal-arista

tests/container_checker/test_container_checker.py

@@ -78,6 +78,11 @@ def check_image_version(duthosts, selected_rand_one_per_hwsku_hostname):
                        "Test was not supported for 201911 and older image version!")
 
 
+def check_monit_running(duthost):
+    monit_services_status = duthost.get_monit_services_status()
+    return monit_services_status
+
+
 @pytest.fixture(autouse=True, scope="module")
 def update_monit_service(duthosts, selected_rand_one_per_hwsku_hostname):
     """Update Monit configuration and restart it.
@@ -99,14 +104,20 @@ def update_monit_service(duthosts, selected_rand_one_per_hwsku_hostname):
         duthost.shell("sudo cp -f /etc/monit/monitrc /tmp/")
         duthost.shell("sudo cp -f /etc/monit/conf.d/sonic-host /tmp/")
 
-        temp_config_line = "    if status != 0 for 1 times within 1 cycles then alert repeat every 1 cycles"
+        cmd_reduce_container_checker_alert_cycle = (
+            "sudo sed -i '/check program container_checker/, /^[ \t]*if status != 0/ { "
+            "/^[ \t]*if status != 0/ { s/^/#/; a\\    if status != 0 "
+            "for 1 times within 1 cycles then alert repeat every 1 cycles\n } } ' "
+            "/etc/monit/conf.d/sonic-host"
+        )
         logger.info("Reduce the monitoring interval of container_checker.")
-        duthost.shell("sudo sed -i '$s/^./#/' /etc/monit/conf.d/sonic-host")
-        duthost.shell("echo '{}' | sudo tee -a /etc/monit/conf.d/sonic-host".format(temp_config_line))
+        duthost.shell(cmd_reduce_container_checker_alert_cycle)
         duthost.shell("sudo sed -i 's/with start delay 300/with start delay 10/' /etc/monit/monitrc")
         duthost.shell("sudo sed -i 's/set daemon 60/set daemon 10/' /etc/monit/monitrc")
         logger.info("Restart the Monit service without delaying to monitor.")
         duthost.shell("sudo systemctl restart monit")
+        is_monit_running = wait_until(320, 5, 5, check_monit_running, duthost)
+        pytest_assert(is_monit_running, "Monit is not running after restart!")
 
     yield