Merge pull request ceph#64385 from rhcs-dashboard/prometheus-http-service-fix

mgr/prometheus: fix metrics service not coming up

Reviewed-by: Adam King <[email protected]>
Reviewed-by: Redouane Kachach <[email protected]>

Author: adk3798

qa/suites/orch/cephadm/workunits/task/test_monitoring_stack_basic.yaml

@@ -64,3 +64,7 @@ tasks:
         curl -s http://${ALERTM_IP}:9093/api/v2/status
         curl -s http://${ALERTM_IP}:9093/api/v2/alerts
         curl -s http://${ALERTM_IP}:9093/api/v2/alerts | jq -e '.[] | select(.labels | .alertname == "CephMonDown") | .status | .state == "active"'
+        # check prometheus metrics endpoint is not empty and make sure we can get metrics
+        METRICS_URL=$(ceph mgr services | jq -r .prometheus)
+        [ -n "$METRICS_URL" ] || exit 1
+        curl -s "${METRICS_URL}metrics" | grep -q '^ceph_health_status'

src/pybind/mgr/prometheus/module.py

@@ -1773,6 +1773,9 @@ def configure(self, server_addr: str, server_port: int) -> None:
                 self.log.exception(f'Failed to setup cephadm based secure monitoring stack: {e}\n',
                                    'Falling back to default configuration')
 
+        # In any error fallback to plain http mode
+        self.setup_default_config(server_addr, server_port)
+
     def setup_default_config(self, server_addr: str, server_port: int) -> None:
         cherrypy.config.update({
             'server.socket_host': server_addr,