Merge pull request #813 from NHSDigital/re-implement-retry-queue

Add retry queue metrics collection to collect_metrics

Author: dnimmo

infrastructure/modules/container-apps/alerts.tf

@@ -62,7 +62,7 @@ resource "azurerm_monitor_scheduled_query_rules_alert_v2" "queue_length_high" {
   criteria {
     query = <<-KQL
       customMetrics
-      | where name == "${each.key}"
+      | where name == "queue_size_${each.key}"
       | extend environment = tostring(customDimensions.environment)
       | where environment == "${var.environment}"
       | extend value = toreal(value)

infrastructure/modules/container-apps/jobs.tf

@@ -46,6 +46,7 @@ locals {
     collect_metrics = {
       cron_expression = "*/5 * * * *"
       environment_variables = {
+        RETRY_QUEUE_NAME          = "notifications-message-batch-retries"
         STATUS_UPDATES_QUEUE_NAME = "notifications-message-status-updates"
         ENVIRONMENT               = var.environment
       }

manage_breast_screening/notifications/management/commands/collect_metrics.py

@@ -11,13 +11,18 @@
 class Command(BaseCommand):
     def handle(self, *args, **options):
         try:
-            queue = Queue.MessageStatusUpdates()
-            Metrics().set_gauge_value(
-                f"queue_size_{queue.queue_name}",
-                "messages",
-                "Queue length",
-                queue.get_message_count(),
-            )
+            queues = [
+                Queue.MessageStatusUpdates(),
+                Queue.RetryMessageBatches(),
+            ]
+
+            for queue in queues:
+                Metrics().set_gauge_value(
+                    f"queue_size_{queue.queue_name}",
+                    "messages",
+                    "Queue length",
+                    queue.get_message_count(),
+                )
 
         except Exception as e:
             logger.error(e, exc_info=True)

manage_breast_screening/notifications/services/queue.py

@@ -69,3 +69,7 @@ def MessageStatusUpdates(cls):
                 "STATUS_UPDATES_QUEUE_NAME", "notifications-message-status-updates"
             )
         )
+
+    @classmethod
+    def RetryMessageBatches(cls):
+        return cls(os.getenv("RETRY_QUEUE_NAME", "notifications-message-batch-retries"))

manage_breast_screening/notifications/tests/management/commands/test_collect_metrics.py

@@ -19,15 +19,27 @@ def test_handle_sends_queue_lengths(self, mock_metrics_class, mock_queue):
         mock_status.queue_name = "status_queue"
         mock_status.get_message_count.return_value = 2
 
+        mock_retry = MagicMock()
+        mock_retry.queue_name = "retry_queue"
+        mock_retry.get_message_count.return_value = 5
+
         mock_queue.MessageStatusUpdates.return_value = mock_status
+        mock_queue.RetryMessageBatches.return_value = mock_retry
 
         Command().handle()
 
         metrics_instance = mock_metrics_class.return_value
 
-        metrics_instance.set_gauge_value.assert_called_once_with(
+        assert metrics_instance.set_gauge_value.call_count == 2
+        metrics_instance.set_gauge_value.assert_any_call(
             "queue_size_status_queue",
             "messages",
             "Queue length",
             2,
         )
+        metrics_instance.set_gauge_value.assert_any_call(
+            "queue_size_retry_queue",
+            "messages",
+            "Queue length",
+            5,
+        )

manage_breast_screening/notifications/tests/services/test_queue.py

@@ -113,6 +113,35 @@ def test_update_queue_prefers_queue_name_from_env(self, monkeypatch):
                 "qqq111", "updates"
             )
 
+    def test_retry_message_batches_queue(self):
+        with patch(
+            "manage_breast_screening.notifications.services.queue.QueueClient"
+        ) as queue_client:
+            mock_client = MagicMock()
+            queue_client.from_connection_string.return_value = mock_client
+
+            Queue.RetryMessageBatches().add("some data")
+
+            queue_client.from_connection_string.assert_called_once_with(
+                "qqq111", "notifications-message-batch-retries"
+            )
+            mock_client.create_queue.assert_called_once()
+            mock_client.send_message.assert_called_once_with("some data")
+
+    def test_retry_queue_prefers_queue_name_from_env(self, monkeypatch):
+        monkeypatch.setenv("RETRY_QUEUE_NAME", "retries")
+        with patch(
+            "manage_breast_screening.notifications.services.queue.QueueClient"
+        ) as queue_client:
+            mock_client = MagicMock()
+            queue_client.from_connection_string.return_value = mock_client
+
+            Queue.RetryMessageBatches()
+
+            queue_client.from_connection_string.assert_called_once_with(
+                "qqq111", "retries"
+            )
+
     def test_queue_raises_configuration_error_when_no_config_provided(
         self, monkeypatch
     ):