Merge branch 'master' into healthz_BE

Author: jayaragini-hcl

host_modules/debug_service.py

@@ -0,0 +1,157 @@
+import pty
+import subprocess
+import os
+import select
+import errno
+import logging
+
+from concurrent.futures import ThreadPoolExecutor, TimeoutError
+from threading import Event
+
+from host_modules import host_service
+
+# Timeout should be slightly less than default DBUS timeout (25 sec)
+TIMEOUT = 20
+MOD_NAME = 'DebugExecutor'
+INTERFACE = host_service.bus_name(MOD_NAME)
+logger = logging.getLogger(__name__)
+
+
+class DebugExecutor(host_service.HostModule):
+    """
+    Debug container command handler.
+    Allows the debug container to execute arbitrary commands on the device, after having been validated against the whitelist.
+    """
+
+    def __init__(self, mod_name):
+        super().__init__(mod_name)
+        self.executor = ThreadPoolExecutor(max_workers=1)
+
+    def _run_and_stream(self, argv, cancellation_event):
+        """
+        Internal method to asynchronously run a command and stream stdout/stderr to the requesting client.
+        """
+        master_fd, slave_fd = pty.openpty()
+
+        # Populate an environment for interactive commands (i.e. 'top')
+        env = os.environ.copy()
+        env['TERM'] = 'xterm'
+
+        p = subprocess.Popen(
+            argv,
+            stdin = slave_fd,
+            stdout = slave_fd,
+            stderr = subprocess.PIPE,
+            close_fds = True,
+            bufsize = 0,
+            universal_newlines = False,
+            env = env,
+        )
+        os.close(slave_fd)
+        if p.stderr == None:
+            raise Exception("Could not open pipe for stderr")
+
+        stderr_fd = p.stderr.fileno()
+        fds = [master_fd, stderr_fd]
+
+        try:
+            while True:
+                ready, _, _ = select.select(fds, [], [])
+
+                # Terminate this process if calling thread exits
+                if cancellation_event.is_set():
+                    break
+
+                if master_fd in ready:
+                    # Master FD is a PTY, will throw exception when closed
+                    try:
+                        data = os.read(master_fd, 4096)
+                        self.Stdout(data.decode(errors='ignore'))
+                    except OSError as e:
+                        if e.errno == errno.EIO:
+                            fds.remove(master_fd)
+                        else:
+                            raise
+
+                if stderr_fd in ready:
+                    # Stderr FD is a normal fd, will be empty when closed
+                    data = os.read(stderr_fd, 4096)
+                    if not data:
+                        fds.remove(stderr_fd)
+                    else:
+                        self.Stderr(data.decode(errors='ignore'))
+
+                if not fds:
+                    break
+
+        finally:
+            os.close(master_fd)
+            os.close(stderr_fd)
+
+            # Check if the process is still running before trying to stop it
+            if p.poll() is None:
+                logger.info(f"Terminating subprocess (PID: {p.pid}) for command '{argv}'...")
+                p.terminate()
+                try:
+                    rc = p.wait(timeout=5)
+                    logger.info(f"Subprocess for '{argv}' terminated gracefully with code: {rc}")
+                except subprocess.TimeoutExpired:
+                    logger.warning(f"Process for '{argv}' did not terminate gracefully. Forcing kill...")
+                    p.kill()
+                    rc = p.wait()
+                    logger.info(f"Subprocess for '{argv}' was forcefully killed, exited with code: {rc}")
+            else:
+                rc = p.poll()
+
+            return rc
+
+    @host_service.signal(INTERFACE, signature='s')
+    def Stdout(self, data):
+        """
+        Signal to emit a line of stdout for a given command.
+        """
+        pass
+
+    @host_service.signal(INTERFACE, signature='s')
+    def Stderr(self, data):
+        """
+        Signal to emit a line of stderr for a given command.
+        """
+        pass
+
+    @host_service.method(INTERFACE, in_signature='as', out_signature='is')
+    def RunCommand(self, argv):
+        """
+        DBus endpoint - receives a command, and streams the response data back to the client.
+        Starts the command in a separate thread, with a timeout once the command has begun execution.
+
+        The thread pool has a limit of 1, to ensure that only one user at a time may execute commands on the device.
+        Additionally, the timeout ensures that commands are stopped once the default DBUS timeout has been reached.
+
+        Returns a tuple, consisting of (int_return_code, string_details)
+        """
+        logger.info(f"Running command: '{argv}'")
+        cancellation_event = Event()
+        future = self.executor.submit(self._run_and_stream, argv, cancellation_event)
+        try:
+            rc = future.result(timeout=TIMEOUT)
+            logger.info(f"Command '{argv}' exited with code: {rc}")
+
+            return (rc, f"Command exited with {rc}")
+        except TimeoutError as e:
+            err_msg = f"TimeoutError: Command '{argv}' took longer than {TIMEOUT} sec to complete"
+            logger.error(err_msg)
+
+            cancellation_event.set()
+
+            return (errno.ETIMEDOUT, err_msg)
+        except Exception as e:
+            exception_type = type(e).__name__
+            err_details = str(e) if str(e) else 'No details within error message'
+
+            err_msg = f"{exception_type}: Command '{argv}' caused exception to be thrown: {err_details}"
+            logger.error(err_msg)
+
+            cancellation_event.set()
+
+            return (errno.EIO, err_msg)

host_modules/gnoi_reset.py

@@ -0,0 +1,147 @@
+"""gNOI reset module which performs factory reset."""
+
+import json
+import logging
+import threading
+import time
+from host_modules import host_service
+from host_modules.reboot import Reboot
+
+MOD_NAME = "gnoi_reset"
+
+logger = logging.getLogger(__name__)
+
+
+class GnoiReset(host_service.HostModule):
+    """DBus endpoint that executes the factory reset and returns the reset status and response."""
+
+    def __init__(self, mod_name):
+        self.lock = threading.Lock()
+        self.is_reset_ongoing = False
+        self.reset_request = {}
+        self.reset_response = {}
+        super(GnoiReset, self).__init__(mod_name)
+
+    def populate_reset_response(
+        self,
+        reset_success=True,
+        factory_os_unsupported=False,
+        zero_fill_unsupported=False,
+        detail="",
+    ) -> tuple[int, str]:
+        """Populate the factory reset response.
+        """
+        with self.lock:
+            self.reset_response = {}
+            response = {}
+            if reset_success:
+                self.reset_response["reset_success"] = {}
+                response["reset_success"] = {}
+            else:
+                self.reset_response["reset_error"] = {}
+                response["reset_error"] = {}
+                if factory_os_unsupported:
+                    self.reset_response["reset_error"]["factory_os_unsupported"] = True
+                elif zero_fill_unsupported:
+                    self.reset_response["reset_error"]["zero_fill_unsupported"] = True
+                else:
+                    self.reset_response["reset_error"]["other"] = True
+                response["reset_error"]["detail"] = detail
+            response_data = json.dumps(response)
+        return 0, response_data
+
+    def _check_reboot_in_progress(self) -> int:
+        """Checks if reboot is already in progress."""
+        if self.is_reset_ongoing:
+            return 1
+        else:
+            return 0
+
+    def _parse_arguments(self, options) -> tuple[int, str]:
+        """Parses and validates the given arguments into a reset request."""
+        try:
+            raw = json.loads(options)
+        except ValueError as e:
+            logger.error("[%s]:Failed to parse factory reset request: %s", MOD_NAME, str(e))
+            return self.populate_reset_response(
+                reset_success=False,
+                detail="Failed to parse json formatted factory reset request into python dict.",
+            )
+
+        # Normalize: support both camelCase and snake_case
+        self.reset_request = {
+            "factoryOs": raw.get("factoryOs", raw.get("factory_os", False)),
+            "zeroFill": raw.get("zeroFill", raw.get("zero_fill", False)),
+            "retainCerts": raw.get("retainCerts", raw.get("retain_certs", False)),
+        }
+
+        # Reject the request if zero_fill is set.
+        if self.reset_request["factoryOs"] and self.reset_request["zeroFill"]:
+            return self.populate_reset_response(
+                reset_success=False,
+                zero_fill_unsupported=True,
+                detail="zero_fill operation is currently unsupported.",
+            )
+        # Issue a warning if retain_certs is set.
+        if self.reset_request["factoryOs"] and self.reset_request["retainCerts"]:
+            logger.warning("%s: retain_certs is currently ignored.", MOD_NAME)
+            return self.populate_reset_response(
+                reset_success=False,
+                detail="Method FactoryReset.Start is currently unsupported."
+            )
+        # Reject the request if factoryOs is set. As the method is currently unsupported 
+        if self.reset_request["factoryOs"]:
+            return self.populate_reset_response(
+                reset_success=False,
+                detail="Method FactoryReset.Start is currently unsupported."
+            )
+
+        # Default fallback if no valid options triggered any action
+        return self.populate_reset_response(
+            reset_success=False,
+            detail="Method FactoryReset.Start is currently unsupported."
+        )
+
+    def _execute_reboot(self) -> int:
+        try:
+            r = Reboot("reboot")
+            t = threading.Thread(target=r.execute_reboot, args=("COLD",))
+            t.start()
+        except RuntimeError:
+            self.is_reset_ongoing = False
+            return 1
+
+        return 0
+
+    @host_service.method(
+        host_service.bus_name(MOD_NAME), in_signature="as", out_signature="is"
+    )
+
+    def issue_reset(self, options) -> tuple[int, str]:
+        """Issues the factory reset."""
+        print("Issuing reset from back end")
+
+        rc, resp = self._parse_arguments(options)
+        if not rc:
+            return rc, resp
+
+        rc = self._check_reboot_in_progress()
+        if rc:
+            return self.populate_reset_response(reset_success=False, detail="Previous reset is ongoing.")
+
+        self.is_reset_ongoing = True
+
+        rc, resp = self._execute_reboot()
+        if rc:
+            return self.populate_reset_response(reset_success=False,detail="Failed to start thread to execute reboot.")
+
+        # Default fallback if no valid options triggered any action
+        return self.populate_reset_response(
+            reset_success=False,
+            detail="Method FactoryReset.Start is currently unsupported."
+        )
+
+
+def register():
+    """Return the class name"""
+    return GnoiReset, MOD_NAME

host_modules/host_service.py

@@ -15,6 +15,7 @@ def bus_path(mod_name):
     return BUS_PATH + '/' + mod_name
 
 method = dbus.service.method
+signal = dbus.service.signal
 
 class HostService(dbus.service.Object):
     """Service class for top level DBus endpoint"""

scripts/determine-reboot-cause

@@ -44,6 +44,7 @@ REBOOT_TYPE_KEXEC_PATTERN_EXPRESS = ".*SONIC_BOOT_TYPE=(express).*"
 REBOOT_CAUSE_UNKNOWN = "Unknown"
 REBOOT_CAUSE_NON_HARDWARE = "Non-Hardware"
 REBOOT_CAUSE_HARDWARE_OTHER = "Hardware - Other"
+REBOOT_CAUSE_HEARTBEAT_LOSS = "Heartbeat with the Supervisor card lost"
 
 # Global logger class instance
 sonic_logger = syslogger.SysLogger(SYSLOG_IDENTIFIER)
@@ -165,6 +166,8 @@ def get_reboot_cause_dict(previous_reboot_cause, comment, gen_time):
         if match is not None:
             reboot_cause_dict['cause'] = "Kernel Panic"
             reboot_cause_dict['time'] = match.group(1)
+    elif re.search(r'Heartbeat with the Supervisor card lost', previous_reboot_cause):
+        reboot_cause_dict['cause'] = 'Heartbeat with the Supervisor card lost'
 
     return reboot_cause_dict
 
@@ -185,7 +188,7 @@ def determine_reboot_cause():
     software_reboot_cause = find_software_reboot_cause()
 
     # The main decision logic of the reboot cause:
-    # If there is a valid hardware reboot cause indicated by platform API,
+    # If software reboot cause is not heartbeat loss and there is a valid hardware reboot cause indicated by platform API,
     #    check the software reboot cause to add additional reboot cause.
     # If there is a reboot cause indicated by /proc/cmdline, and/or warmreboot/fastreboot/softreboot
     #   the software_reboot_cause which is the content of /hosts/reboot-cause/reboot-cause.txt
@@ -194,7 +197,7 @@ def determine_reboot_cause():
     #   the software_reboot_cause will be treated as the reboot cause if it's not unknown
     #   otherwise, the cmdline_reboot_cause will be treated as the reboot cause if it's not none
     # Else the software_reboot_cause will be treated as the reboot cause
-    if REBOOT_CAUSE_NON_HARDWARE not in hardware_reboot_cause:
+    if REBOOT_CAUSE_HEARTBEAT_LOSS not in software_reboot_cause and REBOOT_CAUSE_NON_HARDWARE not in hardware_reboot_cause:
         previous_reboot_cause = hardware_reboot_cause
         # Check if any software reboot was issued before this hardware reboot happened
         if software_reboot_cause is not REBOOT_CAUSE_UNKNOWN:

scripts/hostcfgd

@@ -2177,14 +2177,27 @@ class HostConfigDaemon:
         # Initialize LoggingCfg
         self.loggingcfg = LoggingCfg()
 
-    def load(self, init_data):
+    def load_independent_config(self, init_data):
+        # Load config that does not rely on any services
         aaa = init_data['AAA']
         tacacs_global = init_data['TACPLUS']
         tacacs_server = init_data['TACPLUS_SERVER']
         radius_global = init_data['RADIUS']
         radius_server = init_data['RADIUS_SERVER']
         ldap_global = init_data['LDAP']
         ldap_server = init_data['LDAP_SERVER']
+        self.aaacfg.load(aaa, tacacs_global, tacacs_server, radius_global, radius_server, ldap_global, ldap_server)
+
+    def load(self, init_data):
+        self.load_independent_config(init_data)
+        
+        syslog.syslog(syslog.LOG_INFO,
+                      "Waiting for systemctl to finish initialization")
+        self.wait_till_system_init_done()
+        syslog.syslog(syslog.LOG_INFO,
+                      "systemctl has finished initialization -- proceeding ...")
+
+        # Load configuration that depends on initialized services
         lpbk_table = init_data['LOOPBACK_INTERFACE']
         kdump = init_data['KDUMP']
         passwh = init_data['PASSW_HARDENING']
@@ -2204,7 +2217,6 @@ class HostConfigDaemon:
         banner_messages = init_data.get(swsscommon.CFG_BANNER_MESSAGE_TABLE_NAME)
         logging = init_data.get(swsscommon.CFG_LOGGING_TABLE_NAME, {})
 
-        self.aaacfg.load(aaa, tacacs_global, tacacs_server, radius_global, radius_server, ldap_global, ldap_server)
         self.iptables.load(lpbk_table)
         self.kdumpCfg.load(kdump)
         self.passwcfg.load(passwh)
@@ -2465,12 +2477,6 @@ class HostConfigDaemon:
         self.config_db.subscribe(swsscommon.CFG_LOGGING_TABLE_NAME,
                                  make_callback(self.logging_handler))
 
-        syslog.syslog(syslog.LOG_INFO,
-                      "Waiting for systemctl to finish initialization")
-        self.wait_till_system_init_done()
-        syslog.syslog(syslog.LOG_INFO,
-                      "systemctl has finished initialization -- proceeding ...")
-
     def start(self):
         self.config_db.listen(init_data_handler=self.load)