Merge pull request ceph#55537 from guits/node-proxy-handle-fetch-oob-error

node-proxy: improve http error handling in fetch_oob_details

Author: guits

src/ceph-node-proxy/ceph_node_proxy/main.py

@@ -2,6 +2,7 @@
 from ceph_node_proxy.api import NodeProxyApi
 from ceph_node_proxy.reporter import Reporter
 from ceph_node_proxy.util import Config, get_logger, http_req, write_tmp_file, CONFIG
+from urllib.error import HTTPError
 from typing import Dict, Any, Optional
 
 import argparse
@@ -33,6 +34,8 @@ def __init__(self, **kw: Any) -> None:
         self.cephx = {'cephx': {'name': self.cephx_name,
                                 'secret': self.cephx_secret}}
         self.config = Config('/etc/ceph/node-proxy.yml', config=CONFIG)
+        self.username: str = ''
+        self.password: str = ''
 
     def run(self) -> None:
         self.init()
@@ -44,15 +47,16 @@ def init(self) -> None:
         self.init_api()
 
     def fetch_oob_details(self) -> Dict[str, str]:
-        headers, result, status = http_req(hostname=self.mgr_host,
-                                           port=self.mgr_agent_port,
-                                           data=json.dumps(self.cephx),
-                                           endpoint='/node-proxy/oob',
-                                           ssl_ctx=self.ssl_ctx)
-        if status != 200:
-            msg = f'No out of band tool details could be loaded: {status}, {result}'
+        try:
+            headers, result, status = http_req(hostname=self.mgr_host,
+                                               port=self.mgr_agent_port,
+                                               data=json.dumps(self.cephx),
+                                               endpoint='/node-proxy/oob',
+                                               ssl_ctx=self.ssl_ctx)
+        except HTTPError as e:
+            msg = f'No out of band tool details could be loaded: {e.code}, {e.reason}'
             self.log.debug(msg)
-            raise RuntimeError(msg)
+            raise
 
         result_json = json.loads(result)
         oob_details: Dict[str, str] = {
@@ -64,9 +68,13 @@ def fetch_oob_details(self) -> Dict[str, str]:
         return oob_details
 
     def init_system(self) -> None:
-        oob_details = self.fetch_oob_details()
-        self.username: str = oob_details['username']
-        self.password: str = oob_details['password']
+        try:
+            oob_details = self.fetch_oob_details()
+            self.username = oob_details['username']
+            self.password = oob_details['password']
+        except HTTPError:
+            self.log.warning('No oob details could be loaded, exiting...')
+            raise SystemExit(1)
         try:
             self.system = RedfishDellSystem(host=oob_details['host'],
                                             port=oob_details['port'],

src/ceph-node-proxy/ceph_node_proxy/reporter.py

@@ -63,7 +63,7 @@ def main(self) -> None:
                                 self.system.previous_data = self.system.get_system()
                         else:
                             self.log.debug('no diff, not sending data to the mgr.')
-                    time.sleep(5)
             self.log.debug('lock released in reporter loop.')
+            time.sleep(5)
         self.log.debug('exiting reporter loop.')
         raise SystemExit(0)

src/pybind/mgr/cephadm/module.py

@@ -1716,6 +1716,10 @@ def hardware_powercycle(self, hostname: str, yes_i_really_mean_it: bool = False)
             raise OrchestratorValidationError(f"Can't perform powercycle on node {hostname}: {e}")
         return f'Powercycle scheduled on {hostname}'
 
+    @handle_orch_error
+    def node_proxy_fullreport(self, hostname: Optional[str] = None) -> Dict[str, Any]:
+        return self.node_proxy_cache.fullreport(hostname=hostname)
+
     @handle_orch_error
     def node_proxy_summary(self, hostname: Optional[str] = None) -> Dict[str, Any]:
         return self.node_proxy_cache.summary(hostname=hostname)

src/pybind/mgr/cephadm/serve.py

@@ -888,6 +888,13 @@ def update_progress() -> None:
                         hosts_altered.add(d.hostname)
                         break
 
+                # do not attempt to deploy node-proxy agent when oob details are not provided.
+                if slot.daemon_type == 'node-proxy' and slot.hostname not in self.mgr.node_proxy_cache.oob.keys():
+                    self.log.debug(
+                        f'Not deploying node-proxy agent on {slot.hostname} as oob details are not present.'
+                    )
+                    continue
+
                 # deploy new daemon
                 daemon_id = slot.name
 

src/pybind/mgr/orchestrator/_interface.py

@@ -403,6 +403,14 @@ def node_proxy_summary(self, hostname: Optional[str] = None) -> OrchResult[Dict[
         """
         raise NotImplementedError()
 
+    def node_proxy_fullreport(self, hostname: Optional[str] = None) -> OrchResult[Dict[str, Any]]:
+        """
+        Return node-proxy full report
+
+        :param hostname: hostname
+        """
+        raise NotImplementedError()
+
     def node_proxy_firmwares(self, hostname: Optional[str] = None) -> OrchResult[Dict[str, Any]]:
         """
         Return node-proxy firmwares report

src/pybind/mgr/orchestrator/module.py

@@ -497,6 +497,7 @@ def _hardware_status(self, hostname: Optional[str] = None, _end_positional_: int
         """
         table_heading_mapping = {
             'summary': ['HOST', 'STORAGE', 'CPU', 'NET', 'MEMORY', 'POWER', 'FANS'],
+            'fullreport': [],
             'firmwares': ['HOST', 'COMPONENT', 'NAME', 'DATE', 'VERSION', 'STATUS'],
             'criticals': ['HOST', 'COMPONENT', 'NAME', 'STATUS', 'STATE'],
             'memory': ['HOST', 'NAME', 'STATUS', 'STATE'],
@@ -525,6 +526,15 @@ def _hardware_status(self, hostname: Optional[str] = None, _end_positional_: int
                     row.extend([v['status'][key] for key in ['storage', 'processors', 'network', 'memory', 'power', 'fans']])
                     table.add_row(row)
                 output = table.get_string()
+        elif category == 'fullreport':
+            if hostname is None:
+                output = "Missing host name"
+            elif format != Format.json:
+                output = "fullreport only supports json output"
+            else:
+                completion = self.node_proxy_fullreport(hostname=hostname)
+                fullreport: Dict[str, Any] = raise_if_exception(completion)
+                output = json.dumps(fullreport)
         elif category == 'firmwares':
             output = "Missing host name" if hostname is None else self._firmwares_table(hostname, table, format)
         elif category == 'criticals':