feat: message group id for notification entry points

Author: joybytes

app/notifications/process_notifications.py

@@ -218,7 +218,9 @@ def increment_daily_limit_cache(service_id, notification_type):
         redis_store.incr(cache_key)
 
 
-def send_notification_to_queue_detached(key_type, notification_type, notification_id, queue=None):
+def send_notification_to_queue_detached(
+    key_type, notification_type, notification_id, queue=None, message_group_id=None
+):
     if key_type == KEY_TYPE_TEST:
         queue = QueueNames.RESEARCH_MODE
 
@@ -236,14 +238,20 @@ def send_notification_to_queue_detached(key_type, notification_type, notificatio
         deliver_task = get_pdf_for_templated_letter
 
     try:
-        deliver_task.apply_async([str(notification_id)], queue=queue)
+        deliver_task.apply_async([str(notification_id)], queue=queue, MessageGroupId=message_group_id)
     except Exception:
         dao_delete_notifications_by_id(notification_id)
         raise
 
 
 def send_notification_to_queue(notification, queue=None):
-    send_notification_to_queue_detached(notification.key_type, notification.notification_type, notification.id, queue)
+    send_notification_to_queue_detached(
+        notification.key_type,
+        notification.notification_type,
+        notification.id,
+        queue,
+        message_group_id=str(notification.service_id),
+    )
 
 
 def simulated_recipient(to_address, notification_type):

app/v2/notifications/post_notifications.py

@@ -271,6 +271,7 @@ def process_sms_or_email_notification(
             key_type=api_user.key_type,
             notification_type=notification_type,
             notification_id=notification_id,
+            message_group_id=str(service.id),
         )
     else:
         current_app.logger.info(
@@ -373,12 +374,17 @@ def process_letter_notification(
         postage=postage,
     )
 
-    get_pdf_for_templated_letter.apply_async([str(notification.id)], queue=queue)
+    get_pdf_for_templated_letter.apply_async(
+        [str(notification.id)],
+        queue=queue,
+        MessageGroupId=str(service.id),
+    )
 
     if test_key and current_app.config["TEST_LETTERS_FAKE_DELIVERY"]:
         create_fake_letter_callback.apply_async(
             [notification.id, notification.billable_units, notification.postage],
             queue=queue,
+            MessageGroupId=str(service.id),
         )
 
     resp = create_response_for_post_notification(
@@ -428,10 +434,15 @@ def process_precompiled_letter_notifications(*, letter_data, api_key, service, t
             name=TaskNames.SCAN_FILE,
             kwargs={"filename": filename},
             queue=QueueNames.ANTIVIRUS,
+            MessageGroupId=str(service.id),
         )
     else:
         # stub out antivirus in dev
-        sanitise_letter.apply_async([filename], queue=QueueNames.LETTERS)
+        sanitise_letter.apply_async(
+            [filename],
+            queue=QueueNames.LETTERS,
+            MessageGroupId=str(service.id),
+        )
 
     return resp
 

tests/app/celery/test_letters_pdf_tasks.py

@@ -891,6 +891,7 @@ def test_resanitise_pdf_calls_template_preview_with_letter_details(
             "allow_international_letters": expected_international_letters_allowed,
         },
         queue=QueueNames.SANITISE_LETTERS,
+        MessageGroupId=str(sample_letter_notification.service_id),
     )
 
 

tests/app/delivery/test_send_to_providers.py

@@ -366,7 +366,9 @@ def test_send_email_to_provider_should_call_response_task_if_test_key(sample_ema
     send_to_providers.send_email_to_provider(notification)
 
     assert not app.aws_ses_client.send_email.called
-    app.delivery.send_to_providers.send_email_response.assert_called_once_with(str(reference), "john@smith.com")
+    app.delivery.send_to_providers.send_email_response.assert_called_once_with(
+        str(reference), "john@smith.com", notification.service_id
+    )
     persisted_notification = Notification.query.filter_by(id=notification.id).one()
     assert persisted_notification.to == "john@smith.com"
     assert persisted_notification.template_id == sample_email_template.id

tests/app/notifications/test_process_notification.py

@@ -667,17 +667,19 @@ def test_send_notification_to_queue(
     mocker,
 ):
     mocked = mocker.patch(f"app.celery.{expected_task}.apply_async")
-    Notification = namedtuple("Notification", ["id", "key_type", "notification_type", "created_at"])
+    Notification = namedtuple("Notification", ["id", "key_type", "notification_type", "created_at", "service_id"])
+    service_id = uuid.uuid4()
     notification = Notification(
         id=uuid.uuid4(),
         key_type=key_type,
         notification_type=notification_type,
         created_at=datetime.datetime(2016, 11, 11, 16, 8, 18),
+        service_id=service_id,
     )
 
     send_notification_to_queue(notification=notification, queue=requested_queue)
 
-    mocked.assert_called_once_with([str(notification.id)], queue=expected_queue)
+    mocked.assert_called_once_with([str(notification.id)], queue=expected_queue, MessageGroupId=str(service_id))
 
 
 def test_send_notification_to_queue_throws_exception_deletes_notification(sample_notification, mocker):
@@ -687,7 +689,11 @@ def test_send_notification_to_queue_throws_exception_deletes_notification(sample
     )
     with pytest.raises(Boto3Error):
         send_notification_to_queue(sample_notification)
-    mocked.assert_called_once_with([str(sample_notification.id)], queue="send-sms-tasks")
+    mocked.assert_called_once_with(
+        [str(sample_notification.id)],
+        queue="send-sms-tasks",
+        MessageGroupId=str(sample_notification.service_id),
+    )
 
     assert Notification.query.count() == 0
     assert NotificationHistory.query.count() == 0

tests/app/organisation/test_invite_rest.py

@@ -69,7 +69,11 @@ def test_create_invited_org_user(
     assert notification.personalisation["url"].startswith(expected_start_of_invite_url.format(hostnames=hostnames))
     assert len(notification.personalisation["url"]) > len(expected_start_of_invite_url.format(hostnames=hostnames))
 
-    mocked.assert_called_once_with([(str(notification.id))], queue="notify-internal-tasks")
+    mocked.assert_called_once_with(
+        [(str(notification.id))],
+        queue="notify-internal-tasks",
+        MessageGroupId=str(notification.service_id),
+    )
 
 
 def test_create_invited_user_invalid_email(admin_request, sample_organisation, sample_user, mocker):

tests/app/service_invite/test_service_invite_rest.py

@@ -65,7 +65,11 @@ def test_create_invited_user(
     assert len(notification.personalisation["url"]) > len(expected_start_of_invite_url.format(hostnames=hostnames))
     assert str(notification.template_id) == current_app.config["INVITATION_EMAIL_TEMPLATE_ID"]
 
-    mocked.assert_called_once_with([(str(notification.id))], queue="notify-internal-tasks")
+    mocked.assert_called_once_with(
+        [(str(notification.id))],
+        queue="notify-internal-tasks",
+        MessageGroupId=str(notification.service_id),
+    )
 
 
 def test_create_invited_user_without_auth_type(admin_request, sample_service, mocker, invitation_email_template):

tests/app/user/test_rest.py

@@ -659,7 +659,11 @@ def test_send_user_reset_password_should_send_reset_password_link(
     )
 
     notification = Notification.query.first()
-    mocked.assert_called_once_with([str(notification.id)], queue="notify-internal-tasks")
+    mocked.assert_called_once_with(
+        [str(notification.id)],
+        queue="notify-internal-tasks",
+        MessageGroupId=str(notification.service_id),
+    )
     assert notification.reply_to_text == notify_service.get_default_reply_to_email_address()
 
 
@@ -716,7 +720,11 @@ def test_send_user_reset_password_reset_password_link_contains_redirect_link_if_
 
     notification = Notification.query.first()
     assert "?next=blob" in notification.content
-    mocked.assert_called_once_with([str(notification.id)], queue="notify-internal-tasks")
+    mocked.assert_called_once_with(
+        [str(notification.id)],
+        queue="notify-internal-tasks",
+        MessageGroupId=str(notification.service_id),
+    )
 
 
 def test_send_user_reset_password_should_return_400_when_email_is_missing(admin_request, mocker):
@@ -775,7 +783,11 @@ def test_send_already_registered_email(admin_request, sample_user, already_regis
     )
 
     notification = Notification.query.first()
-    mocked.assert_called_once_with(([str(notification.id)]), queue="notify-internal-tasks")
+    mocked.assert_called_once_with(
+        ([str(notification.id)]),
+        queue="notify-internal-tasks",
+        MessageGroupId=str(notification.service_id),
+    )
     assert notification.reply_to_text == notify_service.get_default_reply_to_email_address()
 
 
@@ -807,7 +819,11 @@ def test_send_user_confirm_new_email_returns_204(
     )
 
     notification = Notification.query.first()
-    mocked.assert_called_once_with(([str(notification.id)]), queue="notify-internal-tasks")
+    mocked.assert_called_once_with(
+        ([str(notification.id)]),
+        queue="notify-internal-tasks",
+        MessageGroupId=str(notification.service_id),
+    )
     assert notification.reply_to_text == notify_service.get_default_reply_to_email_address()
 
 

tests/app/user/test_rest_verify.py

@@ -211,7 +211,9 @@ def test_send_user_sms_code(client, sample_user, sms_code_template, mocker):
     assert notification.reply_to_text == notify_service.get_default_sms_sender()
 
     app.celery.provider_tasks.deliver_sms.apply_async.assert_called_once_with(
-        ([str(notification.id)]), queue="notify-internal-tasks"
+        ([str(notification.id)]),
+        queue="notify-internal-tasks",
+        MessageGroupId=str(notification.service_id),
     )
 
 
@@ -236,7 +238,9 @@ def test_send_user_code_for_sms_with_optional_to_field(client, sample_user, sms_
     notification = Notification.query.first()
     assert notification.to == to_number
     app.celery.provider_tasks.deliver_sms.apply_async.assert_called_once_with(
-        ([str(notification.id)]), queue="notify-internal-tasks"
+        ([str(notification.id)]),
+        queue="notify-internal-tasks",
+        MessageGroupId=str(notification.service_id),
     )
 
 
@@ -307,7 +311,11 @@ def test_send_new_user_email_verification(
     assert resp.status_code == 204
     notification = Notification.query.first()
     assert VerifyCode.query.count() == 0
-    mocked.assert_called_once_with(([str(notification.id)]), queue="notify-internal-tasks")
+    mocked.assert_called_once_with(
+        ([str(notification.id)]),
+        queue="notify-internal-tasks",
+        MessageGroupId=str(notification.service_id),
+    )
     assert notification.reply_to_text == notify_service.get_default_reply_to_email_address()
     assert notification.personalisation["name"] == "Test User"
     assert notification.personalisation["url"].startswith(expected_url_starts_with.format(hostnames=hostnames))
@@ -413,7 +421,11 @@ def test_send_user_email_code(
     assert str(noti.template_id) == current_app.config["EMAIL_2FA_TEMPLATE_ID"]
     assert noti.personalisation["name"] == "Test User"
     assert noti.personalisation["url"].startswith(expected_auth_url.format(hostnames=hostnames))
-    deliver_email.assert_called_once_with([str(noti.id)], queue="notify-internal-tasks")
+    deliver_email.assert_called_once_with(
+        [str(noti.id)],
+        queue="notify-internal-tasks",
+        MessageGroupId=str(noti.service_id),
+    )
 
 
 def test_send_user_email_code_with_urlencoded_next_param(admin_request, mocker, sample_user, email_2fa_code_template):

tests/app/v2/notifications/test_post_letter_notifications.py

@@ -71,7 +71,11 @@ def test_post_letter_notification_returns_201(api_client_request, sample_letter_
     )
     assert not resp_json["scheduled_for"]
     assert not notification.reply_to_text
-    mock.assert_called_once_with([str(notification.id)], queue=QueueNames.CREATE_LETTERS_PDF)
+    mock.assert_called_once_with(
+        [str(notification.id)],
+        queue=QueueNames.CREATE_LETTERS_PDF,
+        MessageGroupId=str(sample_letter_template.service_id),
+    )
 
 
 def test_post_letter_notification_sets_postage(api_client_request, notify_db_session, mocker):
@@ -288,7 +292,11 @@ def test_post_letter_notification_with_test_key_creates_pdf_and_sets_status_to_d
 
     notification = Notification.query.one()
 
-    fake_create_letter_task.assert_called_once_with([str(notification.id)], queue="research-mode-tasks")
+    fake_create_letter_task.assert_called_once_with(
+        [str(notification.id)],
+        queue="research-mode-tasks",
+        MessageGroupId=str(sample_letter_template.service_id),
+    )
     assert not fake_create_dvla_response_task.called
     assert notification.status == NOTIFICATION_DELIVERED
     assert notification.updated_at is not None
@@ -322,7 +330,11 @@ def test_post_letter_notification_with_test_key_creates_pdf_and_sets_status_to_s
 
     notification = Notification.query.one()
 
-    fake_create_letter_task.assert_called_once_with([str(notification.id)], queue="research-mode-tasks")
+    fake_create_letter_task.assert_called_once_with(
+        [str(notification.id)],
+        queue="research-mode-tasks",
+        MessageGroupId=str(sample_letter_template.service_id),
+    )
     assert fake_create_dvla_response_task.called
     assert notification.status == NOTIFICATION_SENDING
 
@@ -563,7 +575,11 @@ def test_post_letter_notification_is_delivered_but_still_creates_pdf_if_in_trial
 
     notification = Notification.query.one()
     assert notification.status == NOTIFICATION_DELIVERED
-    fake_create_letter_task.assert_called_once_with([str(notification.id)], queue="research-mode-tasks")
+    fake_create_letter_task.assert_called_once_with(
+        [str(notification.id)],
+        queue="research-mode-tasks",
+        MessageGroupId=str(sample_trial_letter_template.service_id),
+    )
 
 
 def test_post_letter_notification_is_delivered_and_has_pdf_uploaded_to_test_letters_bucket_using_test_key(