Fix for 22430 - SmartSwitch - Reboot cause for DPUs not updated on complete system reboot (#631)

Author: rameshraghupathy

sonic-chassisd/scripts/chassisd

@@ -80,6 +80,7 @@ CHASSIS_MODULE_REBOOT_TIMESTAMP_FIELD = 'timestamp'
 CHASSIS_MODULE_REBOOT_REBOOT_FIELD = 'reboot'
 DEFAULT_LINECARD_REBOOT_TIMEOUT = 180
 DEFAULT_DPU_REBOOT_TIMEOUT = 360
+MAX_DPU_REBOOT_DURATION = 800
 PLATFORM_ENV_CONF_FILE = "/usr/share/sonic/platform/platform_env.conf"
 PLATFORM_JSON_FILE = "/usr/share/sonic/platform/platform.json"
 
@@ -766,6 +767,25 @@ class SmartSwitchModuleUpdater(ModuleUpdater):
         else:
             return 'empty'
 
+    def retrieve_dpu_reboot_info(self, module):
+        """
+        Retrieve the most recent reboot cause and time from previous-reboot-cause.json.
+        Returns (cause_string, time_string), or (None, None) if unavailable.
+        """
+        try:
+            path = os.path.join(MODULE_REBOOT_CAUSE_DIR, module.lower(), "previous-reboot-cause.json")
+            if os.path.exists(path):
+                with open(path, 'r') as f:
+                    data = json.load(f)
+                    cause = data.get("cause")
+                    time_str = data.get("name")  # Format: "YYYY_MM_DD_HH_MM_SS"
+                    return cause, time_str
+            else:
+                self.log_debug(f"{module}: previous-reboot-cause.json not found")
+        except Exception as e:
+            self.log_error(f"{module}: Failed to read previous-reboot-cause.json: {e}")
+        return None, None
+
     def clear_transition_flag(self, key):
         status, fvs = self.module_table.get(key)
         if status and fvs:
@@ -812,23 +832,44 @@ class SmartSwitchModuleUpdater(ModuleUpdater):
                 current_status = module_info_dict[CHASSIS_MODULE_INFO_OPERSTATUS_FIELD]
 
                 # Operational status transitioning to offline
-                if prev_status != str(ModuleBase.MODULE_STATUS_OFFLINE) and current_status == str(ModuleBase.MODULE_STATUS_OFFLINE):
+                if prev_status != ModuleBase.MODULE_STATUS_EMPTY and prev_status != str(ModuleBase.MODULE_STATUS_OFFLINE) and current_status == str(ModuleBase.MODULE_STATUS_OFFLINE):
                     self.log_notice("{} operational status transitioning to offline".format(key))
 
                     # Persist dpu down time
                     self.persist_dpu_reboot_time(key)
+                    # persist reboot cause
+                    reboot_cause = try_get(self.chassis.get_module(module_index).get_reboot_cause)
+                    self.persist_dpu_reboot_cause(reboot_cause, key)
+                    # publish reboot cause to db
+                    self.update_dpu_reboot_cause_to_db(key)
 
-                    # Clear transition flag in STATE_DB
-                    self.clear_transition_flag(key)
+                elif (prev_status == ModuleBase.MODULE_STATUS_EMPTY or prev_status == str(ModuleBase.MODULE_STATUS_OFFLINE)) and current_status != str(ModuleBase.MODULE_STATUS_OFFLINE):
+                    self.log_notice(f"{key} operational status transitioning to online")
 
-                elif prev_status == str(ModuleBase.MODULE_STATUS_OFFLINE) and current_status != str(ModuleBase.MODULE_STATUS_OFFLINE):
-                    self.log_notice("{} operational status transitioning to online".format(key))
                     reboot_cause = try_get(self.chassis.get_module(module_index).get_reboot_cause)
-
-                    if not self.retrieve_dpu_reboot_time(key) is None or self._is_first_boot(key):
-                        # persist reboot cause
+                    if isinstance(reboot_cause, (tuple, list)):
+                        current_cause = reboot_cause[0]
+                    else:
+                        current_cause = reboot_cause
+
+                    stored_cause, stored_time_str = self.retrieve_dpu_reboot_info(key)
+
+                    is_reboot = False
+                    if current_cause and stored_cause and stored_time_str:
+                        try:
+                            stored_dt = datetime.strptime(stored_time_str, "%Y_%m_%d_%H_%M_%S").replace(tzinfo=timezone.utc)
+                            now = datetime.now(timezone.utc)
+                            delta_sec = (now - stored_dt).total_seconds()
+
+                            if current_cause == stored_cause and delta_sec <  MAX_DPU_REBOOT_DURATION:
+                                self.log_info(f"{key}: is_reboot=True — same reboot cause within {int(delta_sec)}s")
+                                is_reboot = True
+                        except Exception as e:
+                            self.log_error(f"{key}: Reboot cause/time comparison failed: {e}")
+
+                    if not is_reboot and (stored_time_str is not None or self._is_first_boot(key)):
+                        # persist reboot cause and publish to db
                         self.persist_dpu_reboot_cause(reboot_cause, key)
-                        # publish reboot cause to db
                         self.update_dpu_reboot_cause_to_db(key)
 
                     # Clear transition flag in STATE_DB

sonic-chassisd/tests/test_chassisd.py

@@ -223,15 +223,13 @@ def test_smartswitch_moduleupdater_status_transitions():
     module_updater = SmartSwitchModuleUpdater(SYSLOG_IDENTIFIER, chassis)
 
     # Mock dependent methods
-    with patch.object(module_updater, 'retrieve_dpu_reboot_time', return_value="2024-11-19T00:00:00") \
-            as mock_retrieve_reboot_time, \
-         patch.object(module_updater, '_is_first_boot', return_value=False) as mock_is_first_boot, \
-         patch.object(module_updater, 'persist_dpu_reboot_cause') as mock_persist_reboot_cause, \
-         patch.object(module_updater, 'update_dpu_reboot_cause_to_db') as mock_update_reboot_db, \
-         patch("os.makedirs") as mock_makedirs, \
-         patch("builtins.open", mock_open()) as mock_file, \
-         patch.object(module_updater, '_get_history_path',
-                      return_value="/tmp/prev_reboot_time.txt") as mock_get_history_path:
+    with patch.object(module_updater, 'retrieve_dpu_reboot_info', return_value=("Switch rebooted DPU", "2023_01_01_00_00_00")) as mock_reboot_info, \
+        patch.object(module_updater, '_is_first_boot', return_value=False) as mock_is_first_boot, \
+        patch.object(module_updater, 'persist_dpu_reboot_cause') as mock_persist_reboot_cause, \
+        patch.object(module_updater, 'update_dpu_reboot_cause_to_db') as mock_update_reboot_db, \
+        patch("os.makedirs") as mock_makedirs, \
+        patch("builtins.open", mock_open()) as mock_file, \
+        patch.object(module_updater, '_get_history_path', return_value="/tmp/prev_reboot_time.txt") as mock_get_history_path:
 
         # Transition from ONLINE to OFFLINE
         offline_status = ModuleBase.MODULE_STATUS_OFFLINE
@@ -255,6 +253,59 @@ def test_smartswitch_moduleupdater_status_transitions():
         mock_persist_reboot_cause.assert_called_once()
         mock_update_reboot_db.assert_called_once()
 
+def test_online_transition_skips_reboot_update():
+    chassis = MockSmartSwitchChassis()
+    index = 0
+    name = "DPU0"
+    module = MockModule(index, name, "DPU", ModuleBase.MODULE_TYPE_DPU, 0, "SN123")
+    module.set_oper_status(ModuleBase.MODULE_STATUS_OFFLINE)
+    chassis.module_list.append(module)
+
+    updater = SmartSwitchModuleUpdater(SYSLOG_IDENTIFIER, chassis)
+
+    # Mock the module going ONLINE
+    module.set_oper_status(ModuleBase.MODULE_STATUS_ONLINE)
+
+    with patch.object(updater, 'retrieve_dpu_reboot_info',
+                      return_value=("Switch rebooted DPU", datetime.now(timezone.utc).strftime("%Y_%m_%d_%H_%M_%S"))), \
+         patch.object(module, 'get_reboot_cause', return_value="Switch rebooted DPU"), \
+         patch.object(updater, '_is_first_boot', return_value=False), \
+         patch.object(updater, 'persist_dpu_reboot_cause') as mock_persist, \
+         patch.object(updater, 'update_dpu_reboot_cause_to_db') as mock_update, \
+         patch("builtins.open", mock_open()), \
+         patch("os.makedirs"), \
+         patch.object(updater, '_get_history_path', return_value="/tmp/fake.json"):
+
+        updater.module_db_update()
+
+        # Ensure no reboot update due to is_reboot = True
+        mock_persist.assert_not_called()
+        mock_update.assert_not_called()
+
+def test_retrieve_dpu_reboot_info_success():
+    class DummyChassis:
+        def get_num_modules(self): return 0
+        def init_midplane_switch(self): return False
+
+    updater = SmartSwitchModuleUpdater(SYSLOG_IDENTIFIER, DummyChassis())
+    sample_json = {"cause": "Switch rebooted DPU", "name": "2025_06_25_17_18_52"}
+    with patch("os.path.exists", return_value=True), \
+         patch("builtins.open", mock_open(read_data=json.dumps(sample_json))):
+        cause, time_str = updater.retrieve_dpu_reboot_info("dpu0")
+        assert cause == "Switch rebooted DPU"
+        assert time_str == "2025_06_25_17_18_52"
+
+def test_retrieve_dpu_reboot_info_file_missing():
+    class DummyChassis:
+        def get_num_modules(self): return 0
+        def init_midplane_switch(self): return False  # required for SmartSwitchModuleUpdater
+
+    updater = SmartSwitchModuleUpdater(SYSLOG_IDENTIFIER, DummyChassis())
+    with patch("os.path.exists", return_value=False):
+        cause, time_str = updater.retrieve_dpu_reboot_info("dpu0")
+        assert cause is None
+        assert time_str is None
+
 def test_smartswitch_moduleupdater_check_invalid_name():
     chassis = MockSmartSwitchChassis()
     index = 0