Merge pull request ceph#59889 from adk3798/alertmanager-custom-webhook

mgr/cephadm: fix custom alertmanager webhooks

Reviewed-by: John Mulligan <[email protected]>

Author: adk3798

doc/cephadm/services/monitoring.rst

@@ -598,7 +598,7 @@ webhook urls like so:
     service_type: alertmanager
     spec:
       user_data:
-        default_webhook_urls:
+        webhook_urls:
         - "https://foo"
         - "https://bar"
 

src/pybind/mgr/cephadm/services/monitoring.py

@@ -311,7 +311,7 @@ def get_dependencies(cls, mgr: "CephadmOrchestrator",
 
     def generate_config(self, daemon_spec: CephadmDaemonDeploySpec) -> Tuple[Dict[str, Any], List[str]]:
         assert self.TYPE == daemon_spec.daemon_type
-        default_webhook_urls: List[str] = []
+        webhook_urls: List[str] = []
 
         spec = cast(AlertManagerSpec, self.mgr.spec_store[daemon_spec.service_name].spec)
         try:
@@ -321,7 +321,10 @@ def generate_config(self, daemon_spec: CephadmDaemonDeploySpec) -> Tuple[Dict[st
         user_data = spec.user_data
         if 'default_webhook_urls' in user_data and isinstance(
                 user_data['default_webhook_urls'], list):
-            default_webhook_urls.extend(user_data['default_webhook_urls'])
+            webhook_urls.extend(user_data['default_webhook_urls'])
+        if 'webhook_urls' in user_data and isinstance(
+                user_data['webhook_urls'], list):
+            webhook_urls.extend(user_data['webhook_urls'])
 
         security_enabled, mgmt_gw_enabled, oauth2_enabled = self.mgr._get_security_config()
         if mgmt_gw_enabled:
@@ -340,7 +343,7 @@ def generate_config(self, daemon_spec: CephadmDaemonDeploySpec) -> Tuple[Dict[st
         context = {
             'security_enabled': security_enabled,
             'dashboard_urls': dashboard_urls,
-            'default_webhook_urls': default_webhook_urls,
+            'webhook_urls': webhook_urls,
             'snmp_gateway_urls': snmp_gateway_urls,
             'secure': secure,
         }

src/pybind/mgr/cephadm/templates/services/alertmanager/alertmanager.yml.j2

@@ -23,6 +23,14 @@ route:
       group_interval: 10s
       repeat_interval: 1h
       receiver: 'ceph-dashboard'
+{% if webhook_urls %}
+      continue: true
+    - group_by: ['alertname']
+      group_wait: 10s
+      group_interval: 10s
+      repeat_interval: 1h
+      receiver: 'custom-receiver'
+{% endif %}
 {% if snmp_gateway_urls %}
       continue: true
     - receiver: 'snmp-gateway'
@@ -36,7 +44,9 @@ route:
 receivers:
 - name: 'default'
   webhook_configs:
-{% for url in default_webhook_urls %}
+- name: 'custom-receiver'
+  webhook_configs:
+{% for url in webhook_urls %}
   - url: '{{ url }}'
 {% endfor %}
 - name: 'ceph-dashboard'

src/pybind/mgr/cephadm/tests/test_services.py

@@ -567,6 +567,8 @@ def _get_config(self, url: str) -> str:
         receivers:
         - name: 'default'
           webhook_configs:
+        - name: 'custom-receiver'
+          webhook_configs:
         - name: 'ceph-dashboard'
           webhook_configs:
           - url: '{url}/api/prometheus_receiver'
@@ -714,6 +716,8 @@ def test_alertmanager_config_when_mgmt_gw_enabled(self, _get_fqdn, _run_cephadm,
                 receivers:
                 - name: 'default'
                   webhook_configs:
+                - name: 'custom-receiver'
+                  webhook_configs:
                 - name: 'ceph-dashboard'
                   webhook_configs:
                   - url: 'https://host_fqdn:29443/internal/dashboard/api/prometheus_receiver'
@@ -813,6 +817,8 @@ def test_alertmanager_config_security_enabled(self, _get_fqdn, _run_cephadm, cep
                 receivers:
                 - name: 'default'
                   webhook_configs:
+                - name: 'custom-receiver'
+                  webhook_configs:
                 - name: 'ceph-dashboard'
                   webhook_configs:
                   - url: 'http://{fqdn}:8080/api/prometheus_receiver'
@@ -867,6 +873,119 @@ def test_alertmanager_config_security_enabled(self, _get_fqdn, _run_cephadm, cep
                     use_current_daemon_image=False,
                 )
 
+    @pytest.mark.parametrize(
+        "user_data",
+        [
+            ({'webhook_urls': ['http://foo.com:9999', 'http://bar.com:1111']}),
+            ({'default_webhook_urls': ['http://bar.com:9999', 'http://foo.com:1111']}),
+            ({'default_webhook_urls': ['http://bar.com:9999', 'http://foo.com:1111'],
+              'webhook_urls': ['http://foo.com:9999', 'http://bar.com:1111']}),
+        ],
+    )
+    @patch("cephadm.serve.CephadmServe._run_cephadm")
+    @patch("socket.getfqdn")
+    @patch("cephadm.module.CephadmOrchestrator.get_mgr_ip", lambda _: '::1')
+    @patch("cephadm.services.monitoring.password_hash", lambda password: 'alertmanager_password_hash')
+    @patch('cephadm.cert_mgr.CertMgr.get_root_ca', lambda instance: 'cephadm_root_cert')
+    @patch('cephadm.cert_mgr.CertMgr.generate_cert', lambda instance, fqdn, ip: ('mycert', 'mykey'))
+    def test_alertmanager_config_custom_webhook_urls(
+        self,
+        _get_fqdn,
+        _run_cephadm,
+        cephadm_module: CephadmOrchestrator,
+        user_data: Dict[str, List[str]]
+    ):
+        _run_cephadm.side_effect = async_side_effect(('{}', '', 0))
+        cephadm_module.set_store(AlertmanagerService.USER_CFG_KEY, 'alertmanager_user')
+        cephadm_module.set_store(AlertmanagerService.PASS_CFG_KEY, 'alertmanager_plain_password')
+        fqdn = 'host1.test'
+        _get_fqdn.return_value = fqdn
+
+        print(user_data)
+
+        urls = []
+        if 'default_webhook_urls' in user_data:
+            urls += user_data['default_webhook_urls']
+        if 'webhook_urls' in user_data:
+            urls += user_data['webhook_urls']
+        tab_over = ' ' * 18  # since we'll be inserting this into an indented string
+        webhook_configs_str = '\n'.join(f'{tab_over}- url: \'{u}\'' for u in urls)
+
+        with with_host(cephadm_module, 'test'):
+            with with_service(cephadm_module, AlertManagerSpec(user_data=user_data)):
+
+                y = dedent(f"""
+                # This file is generated by cephadm.
+                # See https://prometheus.io/docs/alerting/configuration/ for documentation.
+
+                global:
+                  resolve_timeout: 5m
+                  http_config:
+                    tls_config:
+                      insecure_skip_verify: true
+
+                route:
+                  receiver: 'default'
+                  routes:
+                    - group_by: ['alertname']
+                      group_wait: 10s
+                      group_interval: 10s
+                      repeat_interval: 1h
+                      receiver: 'ceph-dashboard'
+                      continue: true
+                    - group_by: ['alertname']
+                      group_wait: 10s
+                      group_interval: 10s
+                      repeat_interval: 1h
+                      receiver: 'custom-receiver'
+
+                receivers:
+                - name: 'default'
+                  webhook_configs:
+                - name: 'custom-receiver'
+                  webhook_configs:
+{webhook_configs_str}
+                - name: 'ceph-dashboard'
+                  webhook_configs:
+                  - url: 'http://{fqdn}:8080/api/prometheus_receiver'
+                """).lstrip()
+
+                _run_cephadm.assert_called_with(
+                    'test',
+                    "alertmanager.test",
+                    ['_orch', 'deploy'],
+                    [],
+                    stdin=json.dumps({
+                        "fsid": "fsid",
+                        "name": 'alertmanager.test',
+                        "image": '',
+                        "deploy_arguments": [],
+                        "params": {
+                            'tcp_ports': [9093, 9094],
+                        },
+                        "meta": {
+                            'service_name': 'alertmanager',
+                            'ports': [9093, 9094],
+                            'ip': None,
+                            'deployed_by': [],
+                            'rank': None,
+                            'rank_generation': None,
+                            'extra_container_args': None,
+                            'extra_entrypoint_args': None,
+                        },
+                        "config_blobs": {
+                            "files": {
+                                "alertmanager.yml": y,
+                            },
+                            'peers': [],
+                            "use_url_prefix": False,
+                            "ip_to_bind_to": "",
+                        }
+                    }),
+                    use_current_daemon_image=False,
+                    error_ok=True,
+                )
+
     @patch("cephadm.serve.CephadmServe._run_cephadm")
     @patch("socket.getfqdn")
     @patch("cephadm.module.CephadmOrchestrator.get_mgr_ip", lambda _: '::1')

src/python-common/ceph/deployment/service_spec.py

@@ -2516,14 +2516,14 @@ def __init__(self,
         # service_type: alertmanager
         # service_id: xyz
         # user_data:
-        #   default_webhook_urls:
+        #   webhook_urls:
         #   - "https://foo"
         #   - "https://bar"
         #
         # Documentation:
-        # default_webhook_urls - A list of additional URL's that are
-        #                        added to the default receivers'
-        #                        <webhook_configs> configuration.
+        # webhook_urls - A list of additional URL's that are
+        #                added to the default receivers'
+        #                <webhook_configs> configuration.
         self.user_data = user_data or {}
         self.secure = secure
         self.only_bind_port_on_networks = only_bind_port_on_networks