Fix #532: monitor queues health (#542)

* Add endpoint to expose webhooks details

* Send gauge for errors in each webhook

Author: leplatrem

jbi/models.py

@@ -321,14 +321,22 @@ class BugzillaWebhook(BaseModel):
     """Bugzilla Webhook"""
 
     id: int
-    creator: str
     name: str
     url: str
     event: str
     product: str
     component: str
     enabled: bool
     errors: int
+    # Ignored fields:
+    # creator: str
+
+    @property
+    def slug(self):
+        """Return readable identifier"""
+        name = self.name.replace(" ", "-").lower()
+        product = self.product.replace(" ", "-").lower()
+        return f"{self.id}-{name}-{product}"
 
 
 class BugzillaWebhooksResponse(BaseModel):

jbi/router.py

@@ -92,6 +92,12 @@ def get_whiteboard_tags(
     return actions.by_tag
 
 
+@router.get("/bugzilla_webhooks/")
+def get_bugzilla_webhooks():
+    """API for viewing webhooks details"""
+    return bugzilla.get_client().list_webhooks()
+
+
 @router.get("/jira_projects/")
 def get_jira_projects():
     """API for viewing projects that are currently accessible by API"""

jbi/services/bugzilla.py

@@ -7,6 +7,7 @@
 
 import requests
 from pydantic import parse_obj_as
+from statsd.defaults.env import statsd
 
 from jbi import Operation, environment
 from jbi.models import (
@@ -123,7 +124,7 @@ def list_webhooks(self):
             raise BugzillaClientError(
                 f"Unexpected response content from 'GET {url}' (no 'webhooks' field)"
             )
-        return parsed.webhooks
+        return [wh for wh in parsed.webhooks if "/bugzilla_webhook" in wh.url]
 
 
 @lru_cache(maxsize=1)
@@ -143,10 +144,12 @@ def check_health() -> ServiceHealth:
     # and report disabled ones.
     all_webhooks_enabled = False
     if logged_in:
-        webhooks = client.list_webhooks()
-        jbi_webhooks = [wh for wh in webhooks if "/bugzilla_webhook" in wh.url]
+        jbi_webhooks = client.list_webhooks()
         all_webhooks_enabled = len(jbi_webhooks) > 0
         for webhook in jbi_webhooks:
+            # Report errors in each webhook
+            statsd.gauge(f"jbi.bugzilla.webhooks.{webhook.slug}.errors", webhook.errors)
+            # Warn developers when there are errors
             if webhook.errors > 0:
                 logger.warning(
                     "Webhook %s has %s error(s)", webhook.name, webhook.errors

tests/fixtures/factories.py

@@ -6,6 +6,7 @@
     ActionContext,
     BugzillaBug,
     BugzillaComment,
+    BugzillaWebhook,
     BugzillaWebhookEvent,
     BugzillaWebhookEventChange,
     BugzillaWebhookRequest,
@@ -131,3 +132,20 @@ def jira_context_factory(**overrides):
             **overrides,
         }
     )
+
+
+def bugzilla_webhook_factory(**overrides):
+    return BugzillaWebhook.parse_obj(
+        {
+            "component": "General",
+            "creator": "[email protected]",
+            "enabled": True,
+            "errors": 0,
+            "event": "create,change,attachment,comment",
+            "id": 1,
+            "name": "Test Webhooks",
+            "product": "Firefox",
+            "url": "http://server.example.com/bugzilla_webhook",
+            **overrides,
+        }
+    )

tests/unit/services/test_bugzilla.py

@@ -212,7 +212,7 @@ def test_bugzilla_list_webhooks(mocked_responses):
     webhooks = get_client().list_webhooks()
 
     assert len(webhooks) == 1
-    assert webhooks[0].creator == "Bob"
+    assert webhooks[0].event == "create,change,comment"
     assert "/bugzilla_webhook" in webhooks[0].url
 
 

tests/unit/test_router.py

@@ -1,24 +1,14 @@
 import json
 import os
 from datetime import datetime
+from unittest import mock
 
 from fastapi.testclient import TestClient
 
 from jbi.app import app
 from jbi.environment import get_settings
 from jbi.models import BugzillaWebhook, BugzillaWebhookRequest
-
-EXAMPLE_WEBHOOK = BugzillaWebhook(
-    id=0,
-    creator="",
-    name="",
-    url="http://server/bugzilla_webhook",
-    event="create,change,comment",
-    product="Any",
-    component="Any",
-    enabled=True,
-    errors=0,
-)
+from tests.fixtures.factories import bugzilla_webhook_factory
 
 
 def test_read_root(anon_client):
@@ -69,6 +59,22 @@ def test_powered_by_jbi_filtered(exclude_middleware, anon_client):
     assert "DevTest" not in html
 
 
+def test_webhooks_details(anon_client, mocked_bugzilla):
+    mocked_bugzilla.list_webhooks.return_value = [
+        bugzilla_webhook_factory(),
+        bugzilla_webhook_factory(errors=42, enabled=False),
+    ]
+    resp = anon_client.get("/bugzilla_webhooks/")
+
+    wh1, wh2 = resp.json()
+
+    assert "creator" not in wh1
+    assert wh1["enabled"]
+    assert wh1["errors"] == 0
+    assert not wh2["enabled"]
+    assert wh2["errors"] == 42
+
+
 def test_statics_are_served(anon_client):
     resp = anon_client.get("/static/styles.css")
     assert resp.status_code == 200
@@ -195,7 +201,7 @@ def test_read_heartbeat_bugzilla_webhooks_fails(
 ):
     mocked_bugzilla.logged_in.return_value = True
     mocked_bugzilla.list_webhooks.return_value = [
-        EXAMPLE_WEBHOOK.copy(update={"enabled": False})
+        bugzilla_webhook_factory(enabled=False)
     ]
 
     resp = anon_client.get("/__heartbeat__")
@@ -207,6 +213,26 @@ def test_read_heartbeat_bugzilla_webhooks_fails(
     }
 
 
+def test_heartbeat_bugzilla_reports_webhooks_errors(
+    anon_client, mocked_jira, mocked_bugzilla
+):
+    mocked_bugzilla.logged_in.return_value = True
+    mocked_bugzilla.list_webhooks.return_value = [
+        bugzilla_webhook_factory(id=1, errors=0, product="Remote Settings"),
+        bugzilla_webhook_factory(id=2, errors=3, name="Search Toolbar"),
+    ]
+
+    with mock.patch("jbi.services.bugzilla.statsd") as mocked:
+        anon_client.get("/__heartbeat__")
+
+    mocked.gauge.assert_any_call(
+        "jbi.bugzilla.webhooks.1-test-webhooks-remote-settings.errors", 0
+    )
+    mocked.gauge.assert_any_call(
+        "jbi.bugzilla.webhooks.2-search-toolbar-firefox.errors", 3
+    )
+
+
 def test_read_heartbeat_bugzilla_services_fails(
     anon_client, mocked_jira, mocked_bugzilla
 ):
@@ -227,7 +253,7 @@ def test_read_heartbeat_bugzilla_services_fails(
 def test_read_heartbeat_success(anon_client, mocked_jira, mocked_bugzilla):
     """/__heartbeat__ returns 200 when checks succeed."""
     mocked_bugzilla.logged_in.return_value = True
-    mocked_bugzilla.list_webhooks.return_value = [EXAMPLE_WEBHOOK]
+    mocked_bugzilla.list_webhooks.return_value = [bugzilla_webhook_factory()]
     mocked_jira.get_server_info.return_value = {}
     mocked_jira.projects.return_value = [{"key": "DevTest"}]
     mocked_jira.get_project_components.return_value = [{"name": "Main"}]
@@ -298,7 +324,7 @@ def test_jira_heartbeat_missing_permissions(anon_client, mocked_jira, mocked_bug
 
 def test_jira_heartbeat_unknown_components(anon_client, mocked_jira, mocked_bugzilla):
     mocked_bugzilla.logged_in.return_value = True
-    mocked_bugzilla.list_webhooks.return_value = [EXAMPLE_WEBHOOK]
+    mocked_bugzilla.list_webhooks.return_value = [bugzilla_webhook_factory()]
     mocked_jira.get_server_info.return_value = {}
 
     resp = anon_client.get("/__heartbeat__")
@@ -310,7 +336,7 @@ def test_jira_heartbeat_unknown_components(anon_client, mocked_jira, mocked_bugz
 def test_head_heartbeat_success(anon_client, mocked_jira, mocked_bugzilla):
     """/__heartbeat__ support head requests"""
     mocked_bugzilla.logged_in.return_value = True
-    mocked_bugzilla.list_webhooks.return_value = [EXAMPLE_WEBHOOK]
+    mocked_bugzilla.list_webhooks.return_value = [bugzilla_webhook_factory()]
     mocked_jira.get_server_info.return_value = {}
     mocked_jira.projects.return_value = [{"key": "DevTest"}]
     mocked_jira.get_project_components.return_value = [{"name": "Main"}]