[dualtor_neighbor_check] Adjust zero-mac check condition (#3034)

* [dualtor_neighbor_check] Adjust zero-mac check condition

Author: lolyu

scripts/dualtor_neighbor_check.py

@@ -155,7 +155,7 @@
 
 DB_READ_SCRIPT_CONFIG_DB_KEY = "_DUALTOR_NEIGHBOR_CHECK_SCRIPT_SHA1"
 ZERO_MAC = "00:00:00:00:00:00"
-NEIGHBOR_ATTRIBUTES = ["NEIGHBOR", "MAC", "PORT", "MUX_STATE", "IN_MUX_TOGGLE", "NEIGHBOR_IN_ASIC", "TUNNERL_IN_ASIC", "HWSTATUS"]
+NEIGHBOR_ATTRIBUTES = ["NEIGHBOR", "MAC", "PORT", "MUX_STATE", "IN_MUX_TOGGLE", "NEIGHBOR_IN_ASIC", "TUNNEL_IN_ASIC", "HWSTATUS"]
 NOT_AVAILABLE = "N/A"
 
 
@@ -400,9 +400,12 @@ def check_neighbor_consistency(neighbors, mux_states, hw_mux_states, mac_to_port
             continue
 
         check_result["NEIGHBOR_IN_ASIC"] = neighbor_ip in asic_neighs
-        check_result["TUNNERL_IN_ASIC"] = neighbor_ip in asic_route_destinations
+        check_result["TUNNEL_IN_ASIC"] = neighbor_ip in asic_route_destinations
         if is_zero_mac:
-            check_result["HWSTATUS"] = ((not check_result["NEIGHBOR_IN_ASIC"]) and check_result["TUNNERL_IN_ASIC"])
+            # NOTE: for zero-mac neighbors, two situations:
+            # 1. new neighbor just learnt, no neighbor entry in ASIC, tunnel route present in ASIC.
+            # 2. neighbor expired, neighbor entry still present in ASIC, no tunnel route in ASIC.
+            check_result["HWSTATUS"] = check_result["NEIGHBOR_IN_ASIC"] or check_result["TUNNEL_IN_ASIC"]
         else:
             port_name = mac_to_port_name_map[mac]
             # NOTE: mux server ips are always fixed to the mux port
@@ -415,9 +418,9 @@ def check_neighbor_consistency(neighbors, mux_states, hw_mux_states, mac_to_port
             check_result["IN_MUX_TOGGLE"] = mux_state != hw_mux_state
 
             if mux_state == "active":
-                check_result["HWSTATUS"] = (check_result["NEIGHBOR_IN_ASIC"] and (not check_result["TUNNERL_IN_ASIC"]))
+                check_result["HWSTATUS"] = (check_result["NEIGHBOR_IN_ASIC"] and (not check_result["TUNNEL_IN_ASIC"]))
             elif mux_state == "standby":
-                check_result["HWSTATUS"] = ((not check_result["NEIGHBOR_IN_ASIC"]) and check_result["TUNNERL_IN_ASIC"])
+                check_result["HWSTATUS"] = ((not check_result["NEIGHBOR_IN_ASIC"]) and check_result["TUNNEL_IN_ASIC"])
             else:
                 # skip as unknown mux state
                 continue
@@ -442,7 +445,7 @@ def parse_check_results(check_results):
         if not is_zero_mac:
             check_result["IN_MUX_TOGGLE"] = bool_to_yes_no[in_toggle]
         check_result["NEIGHBOR_IN_ASIC"] = bool_to_yes_no[check_result["NEIGHBOR_IN_ASIC"]]
-        check_result["TUNNERL_IN_ASIC"] = bool_to_yes_no[check_result["TUNNERL_IN_ASIC"]]
+        check_result["TUNNEL_IN_ASIC"] = bool_to_yes_no[check_result["TUNNEL_IN_ASIC"]]
         check_result["HWSTATUS"] = bool_to_consistency[hwstatus]
         if (not hwstatus):
             if is_zero_mac:

tests/dualtor_neighbor_check_test.py

@@ -611,3 +611,69 @@ def test_check_neighbor_consistency_zero_mac_neighbor(self, mock_log_functions):
         assert res is True
         mock_log_warn.assert_has_calls(expected_log_warn_calls)
         mock_log_error.assert_not_called()
+
+    def test_check_neighbor_consistency_zero_mac_expired_neighbor(self, mock_log_functions):
+        mock_log_error, mock_log_warn, _, _ = mock_log_functions
+        neighbors = {"192.168.0.102": "00:00:00:00:00:00"}
+        mux_states = {"Ethernet4": "active"}
+        hw_mux_states = {"Ethernet4": "active"}
+        mac_to_port_name_map = {"ee:86:d8:46:7d:01": "Ethernet4"}
+        asic_route_table = []
+        asic_neigh_table = ["{\"ip\":\"192.168.0.102\",\"rif\":\"oid:0x6000000000671\",\"switch_id\":\"oid:0x21000000000000\"}"]
+        mux_server_to_port_map = {"192.168.0.2": "Ethernet4"}
+        expected_output = ["192.168.0.102", "00:00:00:00:00:00", "N/A", "N/A", "N/A", "yes", "no", "consistent"]
+        expected_log_output = tabulate.tabulate(
+            [expected_output],
+            headers=dualtor_neighbor_check.NEIGHBOR_ATTRIBUTES,
+            tablefmt="simple"
+        ).split("\n")
+        expected_log_warn_calls = [call(line) for line in expected_log_output]
+
+        check_results = dualtor_neighbor_check.check_neighbor_consistency(
+            neighbors,
+            mux_states,
+            hw_mux_states,
+            mac_to_port_name_map,
+            asic_route_table,
+            asic_neigh_table,
+            mux_server_to_port_map
+        )
+        res = dualtor_neighbor_check.parse_check_results(check_results)
+
+        assert res is True
+        mock_log_warn.assert_has_calls(expected_log_warn_calls)
+        mock_log_error.assert_not_called()
+
+    def test_check_neighbor_consistency_inconsistent_zero_mac_neighbor(self, mock_log_functions):
+        mock_log_error, mock_log_warn, _, _ = mock_log_functions
+        neighbors = {"192.168.0.102": "00:00:00:00:00:00"}
+        mux_states = {"Ethernet4": "active"}
+        hw_mux_states = {"Ethernet4": "active"}
+        mac_to_port_name_map = {"ee:86:d8:46:7d:01": "Ethernet4"}
+        asic_route_table = []
+        asic_neigh_table = []
+        mux_server_to_port_map = {"192.168.0.2": "Ethernet4"}
+        expected_output = ["192.168.0.102", "00:00:00:00:00:00", "N/A", "N/A", "N/A", "no", "no", "inconsistent"]
+        expected_log_output = tabulate.tabulate(
+            [expected_output],
+            headers=dualtor_neighbor_check.NEIGHBOR_ATTRIBUTES,
+            tablefmt="simple"
+        ).split("\n")
+        expected_log_warn_calls = [call(line) for line in expected_log_output]
+        expected_log_error_calls = [call("Found neighbors that are inconsistent with mux states: %s", ["192.168.0.102"])]
+        expected_log_error_calls.extend([call(line) for line in expected_log_output])
+
+        check_results = dualtor_neighbor_check.check_neighbor_consistency(
+            neighbors,
+            mux_states,
+            hw_mux_states,
+            mac_to_port_name_map,
+            asic_route_table,
+            asic_neigh_table,
+            mux_server_to_port_map
+        )
+        res = dualtor_neighbor_check.parse_check_results(check_results)
+
+        assert res is False
+        mock_log_warn.assert_has_calls(expected_log_warn_calls)
+        mock_log_error.assert_has_calls(expected_log_error_calls)