[Feature][debugcounterorch] Add support for configurable debug drop monitoring feature (#3509)

arista-hpandya · web-flow · commit 1b3f738f34fd · 2025-10-06T09:35:33.000-07:00
* Add support for configurable debug drop monitoring feature Note: This change depends on sonic-net/sonic-swss-common#971 Fixes #3501 HLD: sonic-net/SONiC#1912 What I did Added logic to read configuration from the DEBUG_DROP_MONITOR table. Added logic to register persistent alerts when the conditions are met. Added logic to toggle the feature off if desired on a per-counter level. Why I did it To implement the persistent drop counter monitoring feature which allows users to configure thresholds for drop counters and register alerts when persistent drops are detected. How I verified it Existing unit tests were run using make check to ensure no functionality was affected. New unit tests have been added to verify the functionality. Manual testing was performed on a SONiC switch to verify that the orchagent correctly reads the configuration, generates alerts when thresholds are met, and can be toggled off/on.
diff --git a/orchagent/Makefile.am b/orchagent/Makefile.am
@@ -29,6 +29,7 @@ dist_swss_DATA = \
 		 pfc_restore.lua \
 		 pfc_restore_cisco-8000.lua \
 		 port_rates.lua \
+		 drop_monitor.lua \
 		 watermark_queue.lua \
 		 watermark_pg.lua \
 		 watermark_bufferpool.lua \
diff --git a/orchagent/debugcounterorch.cpp b/orchagent/debugcounterorch.cpp
@@ -37,6 +37,26 @@ DebugCounterOrch::DebugCounterOrch(DBConnector *db, const vector<string>& table_
     publishDropCounterCapabilities();
 
     gPortsOrch->attach(this);
+
+    // Add drop monitor lua script
+    string dropMonitorPluginName = "drop_monitor.lua";
+    string dropMonitorSha;
+
+    try
+    {
+        string dropMonitorLuaScript = swss::loadLuaScript(dropMonitorPluginName);
+        dropMonitorSha = swss::loadRedisScript(m_countersDb.get(), dropMonitorLuaScript);
+    }
+    catch (const runtime_error &e)
+    {
+        SWSS_LOG_ERROR("Drop monitor flex counter group was not set successfully: %s", e.what());
+    }
+
+    setFlexCounterGroupParameter(DEBUG_DROP_MONITOR_FLEX_COUNTER_GROUP,
+                             DEBUG_DROP_MONITOR_FLEX_COUNTER_POLLING_INTERVAL_MS,
+                             STATS_MODE_READ,
+                             PORT_PLUGIN_FIELD,
+                             dropMonitorSha);
 }
 
 DebugCounterOrch::~DebugCounterOrch(void)
@@ -193,6 +213,70 @@ void DebugCounterOrch::doTask(Consumer& consumer)
                 SWSS_LOG_ERROR("Unknown operation type %s\n", op.c_str());
             }
         }
+        else if (table_name == "DEBUG_DROP_MONITOR")
+        {
+            if (op == SET_COMMAND)
+            {
+                if (key == "CONFIG")
+                {
+                    for (const auto& value : values)
+                    {
+                        string config_name = value.first;
+                        string config_value = value.second;
+
+                        // Check the status of the drop counter monitor feature
+                        try
+                        {
+                            if (config_name == "status")
+                            {
+                                if (config_value == "enabled")
+                                {
+                                    debug_monitor_enabled = true;
+                                    string monitored_debug_counter_stat = counterIdsToStr(portDebugMonitorStatIds);
+                                    SWSS_LOG_DEBUG("Enabling debug drop monitor: %s", monitored_debug_counter_stat.c_str());
+                                    setFlexCounterGroupOperation(DEBUG_DROP_MONITOR_FLEX_COUNTER_GROUP, "enable");
+                                    for (auto const &curr : gPortsOrch->getAllPorts())
+                                    {
+                                        string key = string(DEBUG_DROP_MONITOR_FLEX_COUNTER_GROUP) + ":" + sai_serialize_object_id(curr.second.m_port_id);
+                                        startFlexCounterPolling(gSwitchId, key, monitored_debug_counter_stat, PORT_COUNTER_ID_LIST);
+                                    }
+                                }
+                                else if (config_value == "disabled")
+                                {
+                                    debug_monitor_enabled = false;
+                                    SWSS_LOG_DEBUG("Disabling debug drop monitor");
+                                    setFlexCounterGroupOperation(DEBUG_DROP_MONITOR_FLEX_COUNTER_GROUP, "disable");
+                                    for (auto const &curr : gPortsOrch->getAllPorts())
+                                    {
+                                        string key = string(DEBUG_DROP_MONITOR_FLEX_COUNTER_GROUP) + ":" + sai_serialize_object_id(curr.second.m_port_id);
+                                        stopFlexCounterPolling(gSwitchId, key);
+                                    }
+                                }
+                                else
+                                {
+                                    SWSS_LOG_ERROR("The status of drop counter monitor was not recognized: %s. Accepted values are enabled/disabled.", config_value.c_str());
+                                    task_status = task_process_status::task_failed;
+                                }
+                            }
+                            else
+                            {
+                                SWSS_LOG_ERROR("Config for drop counter monitor was not recognized: %s. Accepted values are status.", config_value.c_str());
+                                task_status = task_process_status::task_failed;
+                            }
+                        }
+                        catch(const std::runtime_error& e)
+                        {
+                            SWSS_LOG_ERROR("Encountered an error when updating DEBUG_DROP_MONITOR. config_name: %s, config_value: %s", config_name.c_str(), config_value.c_str());
+                            task_status = task_process_status::task_failed;
+                        }
+                    }
+                }
+            }
+            else
+            {
+                SWSS_LOG_ERROR("Unknown operation type %s\n", op.c_str());
+            }
+        }
         else
         {
             SWSS_LOG_ERROR("Received update from unknown table '%s'", table_name.c_str());
@@ -336,7 +420,7 @@ task_process_status DebugCounterOrch::uninstallDebugCounter(const string& counte
     string counter_type = counter->getCounterType();
     string counter_stat = counter->getDebugCounterSAIStat();
 
-    uninstallDebugFlexCounters(counter_type, counter_stat);
+    uninstallDebugFlexCounters(counter_type, counter_stat, SAI_NULL_OBJECT_ID, counter_name);
 
     if (counter_type == PORT_INGRESS_DROPS || counter_type == PORT_EGRESS_DROPS)
     {
@@ -531,6 +615,11 @@ void DebugCounterOrch::installDebugFlexCounters(const string& counter_type,
     SWSS_LOG_ENTER();
     CounterType flex_counter_type = getFlexCounterType(counter_type);
 
+    // Track the new counter_stat in debug drop monitor
+    portDebugMonitorStatIds.insert(counter_stat);
+    string monitored_debug_counter_stat = counterIdsToStr(portDebugMonitorStatIds);
+    SWSS_LOG_DEBUG("Added %s to: %s", counter_stat.c_str(), monitored_debug_counter_stat.c_str());
+
     if (flex_counter_type == CounterType::SWITCH_DEBUG)
     {
         flex_counter_manager.addFlexCounterStat(gSwitchId, flex_counter_type, counter_stat);
@@ -556,17 +645,34 @@ void DebugCounterOrch::installDebugFlexCounters(const string& counter_type,
                     curr.second.m_port_id,
                     flex_counter_type,
                     counter_stat);
+
+            if (debug_monitor_enabled)
+            {
+                string key = string(DEBUG_DROP_MONITOR_FLEX_COUNTER_GROUP) + ":" + sai_serialize_object_id(curr.second.m_port_id);
+                stopFlexCounterPolling(gSwitchId, key);
+                startFlexCounterPolling(gSwitchId, key, monitored_debug_counter_stat, PORT_COUNTER_ID_LIST);
+            }
         }
     }
 }
 
 void DebugCounterOrch::uninstallDebugFlexCounters(const string& counter_type,
                                                   const string& counter_stat,
-                                                  sai_object_id_t port_id)
+                                                  sai_object_id_t port_id,
+                                                  const string& counter_name)
 {
     SWSS_LOG_ENTER();
     CounterType flex_counter_type = getFlexCounterType(counter_type);
 
+    // Remove the counter_stat from being tracked in debug drop monitor
+    auto counter_stat_iter = portDebugMonitorStatIds.find(counter_stat);
+    portDebugMonitorStatIds.erase(counter_stat_iter);
+    string monitored_debug_counter_stat = counterIdsToStr(portDebugMonitorStatIds);
+    SWSS_LOG_DEBUG("Removed %s from: %s", counter_stat.c_str(), monitored_debug_counter_stat.c_str());
+
+    // Make a vector of keys to delete from COUNTERS_DB, these keys are used by drop counter monitor
+    std::vector<std::string> debug_drop_monitor_stats_fields;
+
     if (flex_counter_type == CounterType::SWITCH_DEBUG)
     {
         flex_counter_manager.removeFlexCounterStat(gSwitchId, flex_counter_type, counter_stat);
@@ -575,6 +681,10 @@ void DebugCounterOrch::uninstallDebugFlexCounters(const string& counter_type,
     {
         for (auto const &curr : gPortsOrch->getAllPorts())
         {
+            // Remove debug counter stat from being tracked by drop counter monitor
+            string key = string(DEBUG_COUNTER_FLEX_COUNTER_GROUP) + ":" + sai_serialize_object_id(curr.second.m_port_id);
+            stopFlexCounterPolling(gSwitchId, key);
+
             if (port_id != SAI_NULL_OBJECT_ID)
             {
                 if (curr.second.m_port_id != port_id)
@@ -592,8 +702,20 @@ void DebugCounterOrch::uninstallDebugFlexCounters(const string& counter_type,
                 curr.second.m_port_id,
                 flex_counter_type,
                 counter_stat);
+
+            debug_drop_monitor_stats_fields.push_back("DEBUG_DROP_MONITOR_STATS|" + counter_name + "|" + curr.first);
+
+            if (debug_monitor_enabled)
+            {
+                string key = string(DEBUG_DROP_MONITOR_FLEX_COUNTER_GROUP) + ":" + sai_serialize_object_id(curr.second.m_port_id);
+                stopFlexCounterPolling(gSwitchId, key);
+                startFlexCounterPolling(gSwitchId, key, monitored_debug_counter_stat, PORT_COUNTER_ID_LIST);
+            }
         }
     }
+
+    // Delete DEBUG_DROP_MONITOR_STATS for this debug counter
+    m_countersDb->del(debug_drop_monitor_stats_fields);
 }
 
 // Debug Counter Initialization Helper Functions START HERE ----------------------------------------
@@ -657,6 +779,11 @@ void DebugCounterOrch::createDropCounter(const string& counter_name, const strin
     }
 }
 
+bool DebugCounterOrch::getDebugMonitorStatus()
+{
+    return debug_monitor_enabled;
+}
+
 // Debug Counter Configuration Helper Functions START HERE -----------------------------------------
 
 // parseDropReasonUpdate takes a key from CONFIG_DB and returns the 1) the counter name being targeted and
@@ -684,5 +811,21 @@ bool DebugCounterOrch::isDropReasonValid(const string& drop_reason) const
     return true;
 }
 
+string DebugCounterOrch::counterIdsToStr(const std::unordered_set<string>& ids) const
+{
+    SWSS_LOG_ENTER();
+    string str;
+
+    for (const auto& i: ids)
+    {
+        str += i + ",";
+    }
 
+    // Remove trailing ','
+    if (!str.empty())
+    {
+        str.pop_back();
+    }
 
+    return str;
+}
diff --git a/orchagent/debugcounterorch.h b/orchagent/debugcounterorch.h
@@ -17,6 +17,8 @@ extern "C" {
 }
 
 #define DEBUG_COUNTER_FLEX_COUNTER_GROUP "DEBUG_COUNTER"
+#define DEBUG_DROP_MONITOR_FLEX_COUNTER_GROUP "DEBUG_MONITOR_COUNTER"
+#define DEBUG_DROP_MONITOR_FLEX_COUNTER_POLLING_INTERVAL_MS "60000"
 
 using DebugCounterMap = std::unordered_map<std::string, std::unique_ptr<DebugCounter>>;
 
@@ -31,6 +33,8 @@ class DebugCounterOrch: public Orch, public Observer
     void doTask(Consumer& consumer);
 
     void update(SubjectType, void *cntx);
+
+    bool getDebugMonitorStatus();
 private:
     // Debug Capability Reporting Functions
     void publishDropCounterCapabilities();
@@ -57,7 +61,8 @@ class DebugCounterOrch: public Orch, public Observer
     void uninstallDebugFlexCounters(
             const std::string& counter_type,
             const std::string& counter_stat,
-            sai_object_id_t port_id = SAI_NULL_OBJECT_ID);
+            sai_object_id_t port_id = SAI_NULL_OBJECT_ID,
+            const std::string& counter_name = "");
 
     // Debug Counter Initialization Helper Functions
     std::string getDebugCounterType(
@@ -74,6 +79,7 @@ class DebugCounterOrch: public Orch, public Observer
             std::string *counter_name,
             std::string *drop_reason) const;
     bool isDropReasonValid(const std::string& drop_reason) const;
+    std::string counterIdsToStr(const std::unordered_set<std::string>& ids) const;
 
     // Data Members
     std::shared_ptr<swss::DBConnector> m_stateDb = nullptr;
@@ -91,6 +97,11 @@ class DebugCounterOrch: public Orch, public Observer
 
     DebugCounterMap debug_counters;
 
+    // portDebugStatIds will store the debug counter stats that have been
+    // configured
+    bool debug_monitor_enabled = false;
+    std::unordered_set<std::string> portDebugMonitorStatIds;
+
     // free_drop_counters are drop counters that have been created by a user
     // that do not have any drop reasons associated with them yet. Because
     // we cannot create a drop counter without any drop reasons, we keep track
diff --git a/orchagent/drop_monitor.lua b/orchagent/drop_monitor.lua
@@ -0,0 +1,88 @@
+-- KEYS - port IDs
+-- ARGV[1] - counters db index
+-- ARGV[2] - counters table name
+-- ARGV[3] - poll time interval (milliseconds)
+
+local counters_db = ARGV[1]
+local config_db = 4
+local debug_drop_monitor_stat_table = 'DEBUG_DROP_MONITOR_STATS'
+local persistent_drop_alert_table = 'PERSISTENT_DROP_ALERTS'
+
+redis.call('SELECT', counters_db)
+
+-- Helper functions
+local function parse_boolean(str) return str == "true" end
+local function parse_number(str) return tonumber(str) or 0 end
+
+-- Get the debug counters and port name map
+local debug_counter_to_port_stat_map = redis.call('HGETALL', "COUNTERS_DEBUG_NAME_PORT_STAT_MAP")
+local debug_counter_to_port_stat_map_len = redis.call('HLEN', "COUNTERS_DEBUG_NAME_PORT_STAT_MAP")
+local port_name_map = redis.call('HGETALL', "COUNTERS_PORT_NAME_MAP")
+local port_name_map_len = redis.call('HLEN', "COUNTERS_PORT_NAME_MAP")
+
+-- Iterate over the debug counter and get their specific configuration
+for debug_counter_index = 1, debug_counter_to_port_stat_map_len, 2 do
+    local debug_counter = debug_counter_to_port_stat_map[debug_counter_index]
+    local debug_counter_stat = debug_counter_to_port_stat_map[debug_counter_index + 1]
+
+    -- Get the configuration of debug counter
+    redis.call('SELECT', config_db)
+    local debug_counter_table = "DEBUG_COUNTER|" .. debug_counter
+    local status = redis.call('HGET', debug_counter_table, 'drop_monitor_status')
+    local drop_count_threshold = parse_number(redis.call('HGET', debug_counter_table, 'drop_count_threshold'))
+    local incident_count_threshold = parse_number(redis.call('HGET', debug_counter_table, 'incident_count_threshold'))
+    local window = parse_number(redis.call('HGET', debug_counter_table, 'window'))
+    redis.call('SELECT', counters_db)
+
+    -- Detect persistent drops if status is enabled
+    if status == 'enabled' then
+        -- Iterate over all ports
+        for port_index = 1, port_name_map_len, 2 do
+            -- Get counter stats
+            local port = port_name_map[port_index]
+            local port_oid = port_name_map[port_index + 1]
+            local counter_stat_map = "COUNTERS:" .. port_oid
+            local current_drop_count = parse_number(redis.call('HGET', counter_stat_map, debug_counter_stat))
+
+            -- Calculate the delta since previous poll
+            local prev_drop_count = parse_number(redis.call('HGET', debug_drop_monitor_stat_table .. '|' .. debug_counter .. '|' .. port, 'prev_drop_count'))
+            local delta_drop_count = current_drop_count - prev_drop_count
+
+            -- Update the previous drop count
+            redis.call('HSET', debug_drop_monitor_stat_table .. '|' .. debug_counter .. '|' .. port, 'prev_drop_count', current_drop_count)
+
+            -- Fetch the current timestamp
+            local time = redis.call('TIME')
+            local curr_unix_timestamp = tonumber(time[1])
+
+            -- Check if drop count is greater than drop count threshold
+            if delta_drop_count > drop_count_threshold then
+                redis.call('RPUSH', debug_drop_monitor_stat_table .. '|' .. debug_counter .. '|' .. port .. '|incidents', curr_unix_timestamp)
+            end
+
+            -- Remove outdated incidents
+            local incident_count = 0
+            local number_of_outdated_incidents = 0
+            local number_of_incidents = redis.call('LLEN', debug_drop_monitor_stat_table .. '|' .. debug_counter .. '|' .. port .. '|incidents')
+            local incident_timestamps = redis.call('LRANGE', debug_drop_monitor_stat_table .. '|' .. debug_counter .. '|' .. port .. '|incidents', 0, number_of_incidents)
+            for incident_index = 1, number_of_incidents do
+                local time_delta = curr_unix_timestamp - incident_timestamps[incident_index]
+                if (time_delta > window) then
+                    number_of_outdated_incidents = number_of_outdated_incidents + 1
+                else
+                    incident_count = incident_count + 1
+                end
+            end
+
+            -- Delete incidents that are outside the window
+            redis.call('LPOP', debug_drop_monitor_stat_table .. '|' .. debug_counter .. '|' .. port .. '|incidents', number_of_outdated_incidents)
+
+            if incident_count > incident_count_threshold then
+                -- Generate alert for persistent drops
+                redis.call('HSET', persistent_drop_alert_table, debug_counter .. '|' .. curr_unix_timestamp, 'Persistent packet drops detected on ' .. port)
+                -- Delete all incidents since a persistent drop alert was issued
+                redis.call('DEL', debug_drop_monitor_stat_table .. '|' .. debug_counter .. '|' .. port .. '|incidents')
+            end
+        end
+    end
+end
diff --git a/orchagent/flexcounterorch.cpp b/orchagent/flexcounterorch.cpp
@@ -64,6 +64,7 @@ unordered_map<string, string> flexCounterGroupMap =
 {
     {"PORT", PORT_STAT_COUNTER_FLEX_COUNTER_GROUP},
     {"PORT_RATES", PORT_RATE_COUNTER_FLEX_COUNTER_GROUP},
+    {"DEBUG_MONITOR_COUNTER", DEBUG_DROP_MONITOR_FLEX_COUNTER_GROUP},
     {"PORT_BUFFER_DROP", PORT_BUFFER_DROP_STAT_FLEX_COUNTER_GROUP},
     {"QUEUE", QUEUE_STAT_COUNTER_FLEX_COUNTER_GROUP},
     {"PFCWD", PFC_WD_FLEX_COUNTER_GROUP},
diff --git a/orchagent/orchdaemon.cpp b/orchagent/orchdaemon.cpp
@@ -419,7 +419,8 @@ bool OrchDaemon::init()
 
     vector<string> debug_counter_tables = {
         CFG_DEBUG_COUNTER_TABLE_NAME,
-        CFG_DEBUG_COUNTER_DROP_REASON_TABLE_NAME
+        CFG_DEBUG_COUNTER_DROP_REASON_TABLE_NAME,
+        CFG_DEBUG_DROP_MONITOR_TABLE_NAME
     };
 
     gDebugCounterOrch = new DebugCounterOrch(m_configDb, debug_counter_tables, 1000);
diff --git a/tests/mock_tests/mock_orchagent_main.h b/tests/mock_tests/mock_orchagent_main.h
@@ -4,6 +4,7 @@
 #include "switchorch.h"
 #include "crmorch.h"
 #include "portsorch.h"
+#include "debugcounterorch.h"
 #include "routeorch.h"
 #include "flowcounterrouteorch.h"
 #include "intfsorch.h"
@@ -53,6 +54,7 @@ extern sai_object_id_t gUnderlayIfId;
 extern SwitchOrch *gSwitchOrch;
 extern CrmOrch *gCrmOrch;
 extern PortsOrch *gPortsOrch;
+extern DebugCounterOrch *gDebugCounterOrch;
 extern FgNhgOrch *gFgNhgOrch;
 extern RouteOrch *gRouteOrch;
 extern FlowCounterRouteOrch *gFlowCounterRouteOrch;
diff --git a/tests/mock_tests/portsorch_ut.cpp b/tests/mock_tests/portsorch_ut.cpp

Original file line number	Diff line number	Diff line change
`@@ -64,6 +64,7 @@ unordered_map<string, string> flexCounterGroupMap =`
`64`	`64`	`{`
`65`	`65`	`{"PORT", PORT_STAT_COUNTER_FLEX_COUNTER_GROUP},`
`66`	`66`	`{"PORT_RATES", PORT_RATE_COUNTER_FLEX_COUNTER_GROUP},`
	`67`	`+ {"DEBUG_MONITOR_COUNTER", DEBUG_DROP_MONITOR_FLEX_COUNTER_GROUP},`
`67`	`68`	`{"PORT_BUFFER_DROP", PORT_BUFFER_DROP_STAT_FLEX_COUNTER_GROUP},`
`68`	`69`	`{"QUEUE", QUEUE_STAT_COUNTER_FLEX_COUNTER_GROUP},`
`69`	`70`	`{"PFCWD", PFC_WD_FLEX_COUNTER_GROUP},`