mgr/cephadm: fix custom alertmanager webhooks

Previously, we put the custom webhooks in the "default"
receiver. This didn't actually work as alertmanager only sends
alerts to the default receiver if the alert does not match
the following routes. This meant if you wanted alerts sent
to the dashboard and also a custom location, you weren't
able to do so with the template cephadm provided.

This also swaps to using "webhook_urls" instead of
"default_webhook_urls" for what should be in the spec,
but "default_webhook_urls" is kept working for backwards
compatability

Fixes: https://tracker.ceph.com/issues/68157

Signed-off-by: Adam King <[email protected]>

Author: adk3798

doc/cephadm/services/monitoring.rst

@@ -598,7 +598,7 @@ webhook urls like so:
     service_type: alertmanager
     spec:
       user_data:
-        default_webhook_urls:
+        webhook_urls:
         - "https://foo"
         - "https://bar"
 

src/pybind/mgr/cephadm/services/monitoring.py

@@ -311,7 +311,7 @@ def get_dependencies(cls, mgr: "CephadmOrchestrator",
 
     def generate_config(self, daemon_spec: CephadmDaemonDeploySpec) -> Tuple[Dict[str, Any], List[str]]:
         assert self.TYPE == daemon_spec.daemon_type
-        default_webhook_urls: List[str] = []
+        webhook_urls: List[str] = []
 
         spec = cast(AlertManagerSpec, self.mgr.spec_store[daemon_spec.service_name].spec)
         try:
@@ -321,7 +321,10 @@ def generate_config(self, daemon_spec: CephadmDaemonDeploySpec) -> Tuple[Dict[st
         user_data = spec.user_data
         if 'default_webhook_urls' in user_data and isinstance(
                 user_data['default_webhook_urls'], list):
-            default_webhook_urls.extend(user_data['default_webhook_urls'])
+            webhook_urls.extend(user_data['default_webhook_urls'])
+        if 'webhook_urls' in user_data and isinstance(
+                user_data['webhook_urls'], list):
+            webhook_urls.extend(user_data['webhook_urls'])
 
         security_enabled, mgmt_gw_enabled, oauth2_enabled = self.mgr._get_security_config()
         if mgmt_gw_enabled:
@@ -340,7 +343,7 @@ def generate_config(self, daemon_spec: CephadmDaemonDeploySpec) -> Tuple[Dict[st
         context = {
             'security_enabled': security_enabled,
             'dashboard_urls': dashboard_urls,
-            'default_webhook_urls': default_webhook_urls,
+            'webhook_urls': webhook_urls,
             'snmp_gateway_urls': snmp_gateway_urls,
             'secure': secure,
         }

src/pybind/mgr/cephadm/templates/services/alertmanager/alertmanager.yml.j2

@@ -23,6 +23,14 @@ route:
       group_interval: 10s
       repeat_interval: 1h
       receiver: 'ceph-dashboard'
+{% if webhook_urls %}
+      continue: true
+    - group_by: ['alertname']
+      group_wait: 10s
+      group_interval: 10s
+      repeat_interval: 1h
+      receiver: 'custom-receiver'
+{% endif %}
 {% if snmp_gateway_urls %}
       continue: true
     - receiver: 'snmp-gateway'
@@ -36,7 +44,9 @@ route:
 receivers:
 - name: 'default'
   webhook_configs:
-{% for url in default_webhook_urls %}
+- name: 'custom-receiver'
+  webhook_configs:
+{% for url in webhook_urls %}
   - url: '{{ url }}'
 {% endfor %}
 - name: 'ceph-dashboard'

src/pybind/mgr/cephadm/tests/test_services.py

@@ -564,6 +564,8 @@ def _get_config(self, url: str) -> str:
         receivers:
         - name: 'default'
           webhook_configs:
+        - name: 'custom-receiver'
+          webhook_configs:
         - name: 'ceph-dashboard'
           webhook_configs:
           - url: '{url}/api/prometheus_receiver'
@@ -711,6 +713,8 @@ def test_alertmanager_config_when_mgmt_gw_enabled(self, _get_fqdn, _run_cephadm,
                 receivers:
                 - name: 'default'
                   webhook_configs:
+                - name: 'custom-receiver'
+                  webhook_configs:
                 - name: 'ceph-dashboard'
                   webhook_configs:
                   - url: 'https://host_fqdn:29443/internal/dashboard/api/prometheus_receiver'
@@ -810,6 +814,8 @@ def test_alertmanager_config_security_enabled(self, _get_fqdn, _run_cephadm, cep
                 receivers:
                 - name: 'default'
                   webhook_configs:
+                - name: 'custom-receiver'
+                  webhook_configs:
                 - name: 'ceph-dashboard'
                   webhook_configs:
                   - url: 'http://{fqdn}:8080/api/prometheus_receiver'
@@ -864,6 +870,119 @@ def test_alertmanager_config_security_enabled(self, _get_fqdn, _run_cephadm, cep
                     use_current_daemon_image=False,
                 )
 
+    @pytest.mark.parametrize(
+        "user_data",
+        [
+            ({'webhook_urls': ['http://foo.com:9999', 'http://bar.com:1111']}),
+            ({'default_webhook_urls': ['http://bar.com:9999', 'http://foo.com:1111']}),
+            ({'default_webhook_urls': ['http://bar.com:9999', 'http://foo.com:1111'],
+              'webhook_urls': ['http://foo.com:9999', 'http://bar.com:1111']}),
+        ],
+    )
+    @patch("cephadm.serve.CephadmServe._run_cephadm")
+    @patch("socket.getfqdn")
+    @patch("cephadm.module.CephadmOrchestrator.get_mgr_ip", lambda _: '::1')
+    @patch("cephadm.services.monitoring.password_hash", lambda password: 'alertmanager_password_hash')
+    @patch('cephadm.cert_mgr.CertMgr.get_root_ca', lambda instance: 'cephadm_root_cert')
+    @patch('cephadm.cert_mgr.CertMgr.generate_cert', lambda instance, fqdn, ip: ('mycert', 'mykey'))
+    def test_alertmanager_config_custom_webhook_urls(
+        self,
+        _get_fqdn,
+        _run_cephadm,
+        cephadm_module: CephadmOrchestrator,
+        user_data: Dict[str, List[str]]
+    ):
+        _run_cephadm.side_effect = async_side_effect(('{}', '', 0))
+        cephadm_module.set_store(AlertmanagerService.USER_CFG_KEY, 'alertmanager_user')
+        cephadm_module.set_store(AlertmanagerService.PASS_CFG_KEY, 'alertmanager_plain_password')
+        fqdn = 'host1.test'
+        _get_fqdn.return_value = fqdn
+
+        print(user_data)
+
+        urls = []
+        if 'default_webhook_urls' in user_data:
+            urls += user_data['default_webhook_urls']
+        if 'webhook_urls' in user_data:
+            urls += user_data['webhook_urls']
+        tab_over = ' ' * 18  # since we'll be inserting this into an indented string
+        webhook_configs_str = '\n'.join(f'{tab_over}- url: \'{u}\'' for u in urls)
+
+        with with_host(cephadm_module, 'test'):
+            with with_service(cephadm_module, AlertManagerSpec(user_data=user_data)):
+
+                y = dedent(f"""
+                # This file is generated by cephadm.
+                # See https://prometheus.io/docs/alerting/configuration/ for documentation.
+
+                global:
+                  resolve_timeout: 5m
+                  http_config:
+                    tls_config:
+                      insecure_skip_verify: true
+
+                route:
+                  receiver: 'default'
+                  routes:
+                    - group_by: ['alertname']
+                      group_wait: 10s
+                      group_interval: 10s
+                      repeat_interval: 1h
+                      receiver: 'ceph-dashboard'
+                      continue: true
+                    - group_by: ['alertname']
+                      group_wait: 10s
+                      group_interval: 10s
+                      repeat_interval: 1h
+                      receiver: 'custom-receiver'
+
+                receivers:
+                - name: 'default'
+                  webhook_configs:
+                - name: 'custom-receiver'
+                  webhook_configs:
+{webhook_configs_str}
+                - name: 'ceph-dashboard'
+                  webhook_configs:
+                  - url: 'http://{fqdn}:8080/api/prometheus_receiver'
+                """).lstrip()
+
+                _run_cephadm.assert_called_with(
+                    'test',
+                    "alertmanager.test",
+                    ['_orch', 'deploy'],
+                    [],
+                    stdin=json.dumps({
+                        "fsid": "fsid",
+                        "name": 'alertmanager.test',
+                        "image": '',
+                        "deploy_arguments": [],
+                        "params": {
+                            'tcp_ports': [9093, 9094],
+                        },
+                        "meta": {
+                            'service_name': 'alertmanager',
+                            'ports': [9093, 9094],
+                            'ip': None,
+                            'deployed_by': [],
+                            'rank': None,
+                            'rank_generation': None,
+                            'extra_container_args': None,
+                            'extra_entrypoint_args': None,
+                        },
+                        "config_blobs": {
+                            "files": {
+                                "alertmanager.yml": y,
+                            },
+                            'peers': [],
+                            "use_url_prefix": False,
+                            "ip_to_bind_to": "",
+                        }
+                    }),
+                    use_current_daemon_image=False,
+                    error_ok=True,
+                )
+
     @patch("cephadm.serve.CephadmServe._run_cephadm")
     @patch("socket.getfqdn")
     @patch("cephadm.module.CephadmOrchestrator.get_mgr_ip", lambda _: '::1')

src/python-common/ceph/deployment/service_spec.py

@@ -2479,14 +2479,14 @@ def __init__(self,
         # service_type: alertmanager
         # service_id: xyz
         # user_data:
-        #   default_webhook_urls:
+        #   webhook_urls:
         #   - "https://foo"
         #   - "https://bar"
         #
         # Documentation:
-        # default_webhook_urls - A list of additional URL's that are
-        #                        added to the default receivers'
-        #                        <webhook_configs> configuration.
+        # webhook_urls - A list of additional URL's that are
+        #                added to the default receivers'
+        #                <webhook_configs> configuration.
         self.user_data = user_data or {}
         self.secure = secure
         self.only_bind_port_on_networks = only_bind_port_on_networks