Merge aggregate and reconciliation reports

manage_breast_screening/notifications/queries/reconciliation.sql

@@ -1,9 +1,48 @@
-SELECT    clin.code AS "Clinic code",
-          appt.episode_type AS "Episode type",
-          appt.status AS "Status",
-          COUNT(appt.id) AS "Count"
-FROM      notifications_appointment appt
-JOIN      notifications_clinic clin ON clin.id = appt.clinic_id
-WHERE     appt.created_at::date = %s
-AND       clin.bso_code = %s
-GROUP BY  clin.code, appt.episode_type, appt.status
+SELECT          appt.nhs_number::text AS "NHS number",
+                clin.name || ' (' || clin.code || ')' AS "Clinic name and code",
+                CASE
+                  WHEN appt.episode_type = 'F' THEN 'Routine first call'
+                  WHEN appt.episode_type = 'G' THEN 'GP Referral'
+                  WHEN appt.episode_type = 'H' THEN 'Very high risk'
+                  WHEN appt.episode_type = 'N' THEN 'Early recall'
+                  WHEN appt.episode_type = 'R' THEN 'Routine recall'
+                  WHEN appt.episode_type = 'S' THEN 'Self referral'
+                  WHEN appt.episode_type = 'T' THEN 'VHR short-term recall'
+                END AS "Episode type",
+                CASE
+                  WHEN appt.status = 'B' THEN 'Booked'
+                  WHEN appt.status = 'C' THEN 'Cancelled'
+                END AS "Appointment status",
+                CASE
+                  WHEN let_fld.id IS NOT NULL  THEN 'Failed'
+                  WHEN msg.sent_at IS NOT NULL THEN 'Notified'
+                  ELSE 'Pending'
+                END AS "Notification status",
+                TO_CHAR(appt.created_at, 'yyyy-mm-dd HH24:MI') AS "Appointment created at",
+                TO_CHAR(appt.starts_at, 'yyyy-mm-dd HH24:MI') AS "Appointment date and time",
+                TO_CHAR(appt.cancelled_at, 'yyyy-mm-dd HH24:MI') AS "Appointment cancelled at",
+                TO_CHAR(msg.sent_at, 'yyyy-mm-dd HH24:MI') AS "Notification sent at",
+                TO_CHAR(nhs_sts.status_updated_at, 'yyyy-mm-dd HH24:MI') AS "NHSApp message read at",
+                TO_CHAR(sms_sts.status_updated_at, 'yyyy-mm-dd HH24:MI') AS "SMS delivered at",
+                TO_CHAR(let_sts.status_updated_at, 'yyyy-mm-dd HH24:MI') AS "Letter sent at"
+FROM            notifications_appointment appt
+JOIN            notifications_clinic clin ON clin.id = appt.clinic_id
+LEFT JOIN       notifications_message msg ON msg.appointment_id = appt.id
+LEFT OUTER JOIN notifications_channelstatus nhs_sts ON nhs_sts.message_id = msg.id
+AND             nhs_sts.channel = 'nhsapp'
+AND             nhs_sts.status = 'read'
+LEFT OUTER JOIN notifications_channelstatus sms_sts ON sms_sts.message_id = msg.id
+AND             sms_sts.channel = 'sms'
+AND             sms_sts.status = 'delivered'
+LEFT OUTER JOIN notifications_channelstatus let_sts ON let_sts.message_id = msg.id
+AND             let_sts.channel = 'letter'
+AND             let_sts.status = 'received'
+LEFT OUTER JOIN notifications_channelstatus let_fld ON let_fld.message_id = msg.id
+AND             let_fld.channel = 'letter'
+AND             let_fld.status IN (
+                    'cancelled', 'virus_scan_failed', 'validation_failed',
+                    'technical_failure', 'permanent_failure'
+                  )
+WHERE           appt.created_at::date > CURRENT_DATE::date - INTERVAL %s
+AND             clin.bso_code = %s
+ORDER BY        appt.created_at ASC

manage_breast_screening/notifications/tests/factories.py

@@ -26,11 +26,11 @@ class Meta:
     clinic = SubFactory(ClinicFactory)
     nhs_number = Sequence(lambda n: int("999%07d" % n))
     batch_id = Sequence(lambda n: "AAA%06d" % n)
-    starts_at = datetime.now(tz=models.ZONE_INFO) + timedelta(weeks=4, days=4)
+    starts_at = datetime.now(tz=models.ZONE_INFO) + timedelta(weeks=4)
     status = "B"
     nbss_id = Sequence(lambda n: int("123%06d" % n))
     episode_type = "S"
-    episode_started_at = datetime.now(tz=models.ZONE_INFO) - timedelta(weeks=5, days=5)
+    episode_started_at = datetime.now(tz=models.ZONE_INFO) - timedelta(weeks=5)
     assessment = False
     number = "1"
 

manage_breast_screening/notifications/tests/queries/test_reconciliation_query.py

@@ -1,84 +1,221 @@
+from collections import namedtuple
 from datetime import datetime, timedelta
 
 import pytest
+import time_machine
 
-from manage_breast_screening.notifications.models import ZONE_INFO
+from manage_breast_screening.notifications.models import ZONE_INFO, Clinic
 from manage_breast_screening.notifications.queries.helper import Helper
 from manage_breast_screening.notifications.tests.factories import (
     AppointmentFactory,
+    ChannelStatusFactory,
     ClinicFactory,
+    MessageFactory,
+)
+
+
+def formatted(dt: datetime):
+    return dt.strftime("%Y-%m-%d %H:%M")
+
+
+def days_time(days: int):
+    return datetime.now(tz=ZONE_INFO) + timedelta(days=days)
+
+
+def create_appointment_set(
+    nhs_number: str,
+    date: datetime,
+    clinic: Clinic,
+    appointment_status: str,
+    episode_type: str,
+    message_status: str | None = None,
+    channel_statuses: dict[str, str] | None = None,
+):
+    now = datetime.now(tz=ZONE_INFO)
+    appt = AppointmentFactory(
+        clinic=clinic,
+        status=appointment_status,
+        episode_type=episode_type,
+        nhs_number=nhs_number,
+        starts_at=date,
+    )
+    if appointment_status == "C":
+        appt.cancelled_at = now
+        appt.save()
+
+    if message_status:
+        message = MessageFactory(appointment=appt, status=message_status, sent_at=now)
+        for channel, status in channel_statuses.items():
+            ChannelStatusFactory(
+                message=message,
+                channel=channel,
+                status=status,
+                status_updated_at=now,
+            )
+    return appt
+
+
+ResultRow = namedtuple(
+    "ResultRow",
+    [
+        "nhs_number",
+        "clinic",
+        "episode_type",
+        "status",
+        "message_status",
+        "created_at",
+        "appointment_starts_at",
+        "cancelled_at",
+        "message_sent_at",
+        "nhs_app_read_at",
+        "sms_delivered_at",
+        "letter_sent_at",
+    ],
 )
 
 
 @pytest.mark.django_db
 class TestReconciliationQuery:
-    def test_appointments_for_today_by_episode_type(self):
-        clinic1 = ClinicFactory(bso_code="MDB", code="BU001")
-        clinic2 = ClinicFactory(bso_code="MDB", code="BU002")
-        clinic3 = ClinicFactory(bso_code="MDB", code="BU003")
-
-        AppointmentFactory.create_batch(size=7, clinic=clinic1)
-        AppointmentFactory.create_batch(size=8, clinic=clinic1, episode_type="R")
-        AppointmentFactory.create_batch(size=2, clinic=clinic1, episode_type="G")
-        AppointmentFactory.create_batch(
-            size=2, clinic=clinic1, episode_type="G", status="C"
-        )
-        AppointmentFactory.create_batch(size=3, clinic=clinic1, episode_type="F")
-        AppointmentFactory.create_batch(size=5, clinic=clinic1, episode_type="H")
-        AppointmentFactory.create_batch(size=2, clinic=clinic2, episode_type="F")
-        AppointmentFactory.create_batch(
-            size=3, clinic=clinic2, episode_type="F", status="A"
-        )
-        AppointmentFactory.create_batch(size=9, clinic=clinic2, episode_type="H")
-        AppointmentFactory.create_batch(size=5, clinic=clinic2, episode_type="N")
-        AppointmentFactory.create_batch(size=2, clinic=clinic3)
-        AppointmentFactory.create_batch(size=5, clinic=clinic3, episode_type="F")
-        AppointmentFactory.create_batch(size=4, clinic=clinic3, episode_type="G")
-        AppointmentFactory.create_batch(size=6, clinic=clinic3, episode_type="R")
-
-        tomorrow_appt = AppointmentFactory(clinic=clinic1)
-        tomorrow_appt.created_at = datetime.now(tz=ZONE_INFO) + timedelta(days=1)
-        tomorrow_appt.save()
-
-        results = Helper.fetchall(
-            "reconciliation",
-            (
-                datetime.now(tz=ZONE_INFO).date(),
-                "MDB",
-            ),
-        )
-
-        assert len(results) == 14
-
-        assert results[0] == ("BU001", "F", "B", 3)
-        assert results[1] == ("BU001", "G", "B", 2)
-        assert results[2] == ("BU001", "G", "C", 2)
-        assert results[3] == ("BU001", "H", "B", 5)
-        assert results[4] == ("BU001", "R", "B", 8)
-        assert results[5] == ("BU001", "S", "B", 7)
-
-        assert results[6] == ("BU002", "F", "A", 3)
-        assert results[7] == ("BU002", "F", "B", 2)
-        assert results[8] == ("BU002", "H", "B", 9)
-        assert results[9] == ("BU002", "N", "B", 5)
-
-        assert results[10] == ("BU003", "F", "B", 5)
-        assert results[11] == ("BU003", "G", "B", 4)
-        assert results[12] == ("BU003", "R", "B", 6)
-        assert results[13] == ("BU003", "S", "B", 2)
+    @pytest.fixture(autouse=True)
+    def setup(self):
+        clinic1 = ClinicFactory(code="BU001", bso_code="BSO1", name="BSU 1")
+        clinic2 = ClinicFactory(code="BU002", bso_code="BSO1", name="BSU 2")
+
+        nhsapp_read = {"nhsapp": "read"}
+        sms_delivered = {"nhsapp": "unnotified", "sms": "delivered"}
+        letter_sent = {
+            "nhsapp": "unnotified",
+            "sms": "permanent_failure",
+            "letter": "received",
+        }
+        failed = {
+            "nhsapp": "unnotified",
+            "sms": "permanent_failure",
+            "letter": "validation_failed",
+        }
+
+        test_data = [
+            ["9991112211", days_time(4), clinic1, "B", "R"],
+            ["9991112214", days_time(6), clinic2, "C", "G"],
+            ["9991112221", days_time(5), clinic2, "B", "R", "failed", failed],
+            ["9991112222", days_time(6), clinic1, "B", "S", "delivered", nhsapp_read],
+            ["9991112223", days_time(5), clinic2, "B", "S", "delivered", sms_delivered],
+            ["9991112229", days_time(5), clinic1, "B", "R", "delivered", letter_sent],
+            ["9991112252", days_time(6), clinic2, "C", "S", "delivered", nhsapp_read],
+        ]
+
+        for d in test_data:
+            create_appointment_set(*d)
+
+    @time_machine.travel(datetime.now(tz=ZONE_INFO), tick=False)
+    def test_appointments_with_various_delivery_states(self):
+        now = datetime.now(tz=ZONE_INFO)
+        results = Helper.fetchall("reconciliation", ["1 week", "BSO1"])
+
+        r = ResultRow(*results[0])
+        assert r.nhs_number == "9991112211"
+        assert r.clinic == "BSU 1 (BU001)"
+        assert r.episode_type == "Routine recall"
+        assert r.status == "Booked"
+        assert r.message_status == "Pending"
+        assert r.created_at == formatted(now)
+        assert r.appointment_starts_at == formatted(days_time(4))
+        assert r.cancelled_at is None
+        assert r.message_sent_at is None
+        assert r.nhs_app_read_at is None
+        assert r.sms_delivered_at is None
+        assert r.letter_sent_at is None
+
+        r = ResultRow(*results[1])
+        assert r.nhs_number == "9991112214"
+        assert r.clinic == "BSU 2 (BU002)"
+        assert r.episode_type == "GP Referral"
+        assert r.status == "Cancelled"
+        assert r.message_status == "Pending"
+        assert r.created_at == formatted(now)
+        assert r.appointment_starts_at == formatted(days_time(6))
+        assert r.cancelled_at == formatted(now)
+        assert r.message_sent_at is None
+        assert r.nhs_app_read_at is None
+        assert r.sms_delivered_at is None
+        assert r.letter_sent_at is None
+
+        r = ResultRow(*results[2])
+        assert r.nhs_number == "9991112221"
+        assert r.clinic == "BSU 2 (BU002)"
+        assert r.episode_type == "Routine recall"
+        assert r.status == "Booked"
+        assert r.message_status == "Failed"
+        assert r.created_at == formatted(now)
+        assert r.appointment_starts_at == formatted(days_time(5))
+        assert r.cancelled_at is None
+        assert r.message_sent_at == formatted(now)
+        assert r.nhs_app_read_at is None
+        assert r.sms_delivered_at is None
+        assert r.letter_sent_at is None
+
+        r = ResultRow(*results[3])
+        assert r.nhs_number == "9991112222"
+        assert r.clinic == "BSU 1 (BU001)"
+        assert r.episode_type == "Self referral"
+        assert r.status == "Booked"
+        assert r.message_status == "Notified"
+        assert r.created_at == formatted(now)
+        assert r.appointment_starts_at == formatted(days_time(6))
+        assert r.cancelled_at is None
+        assert r.message_sent_at == formatted(now)
+        assert r.nhs_app_read_at == formatted(now)
+        assert r.sms_delivered_at is None
+        assert r.letter_sent_at is None
+
+        r = ResultRow(*results[4])
+        assert r.nhs_number == "9991112223"
+        assert r.clinic == "BSU 2 (BU002)"
+        assert r.episode_type == "Self referral"
+        assert r.status == "Booked"
+        assert r.message_status == "Notified"
+        assert r.created_at == formatted(now)
+        assert r.appointment_starts_at == formatted(days_time(5))
+        assert r.cancelled_at is None
+        assert r.message_sent_at == formatted(now)
+        assert r.nhs_app_read_at is None
+        assert r.sms_delivered_at == formatted(now)
+        assert r.letter_sent_at is None
+
+        r = ResultRow(*results[5])
+        assert r.nhs_number == "9991112229"
+        assert r.clinic == "BSU 1 (BU001)"
+        assert r.episode_type == "Routine recall"
+        assert r.status == "Booked"
+        assert r.message_status == "Notified"
+        assert r.created_at == formatted(now)
+        assert r.appointment_starts_at == formatted(days_time(5))
+        assert r.cancelled_at is None
+        assert r.message_sent_at == formatted(now)
+        assert r.nhs_app_read_at is None
+        assert r.sms_delivered_at is None
+        assert r.letter_sent_at == formatted(now)
+
+        r = ResultRow(*results[6])
+        assert r.nhs_number == "9991112252"
+        assert r.clinic == "BSU 2 (BU002)"
+        assert r.episode_type == "Self referral"
+        assert r.status == "Cancelled"
+        assert r.message_status == "Notified"
+        assert r.created_at == formatted(now)
+        assert r.appointment_starts_at == formatted(days_time(6))
+        assert r.cancelled_at == formatted(now)
+        assert r.message_sent_at == formatted(now)
+        assert r.nhs_app_read_at == formatted(now)
+        assert r.sms_delivered_at is None
+        assert r.letter_sent_at is None
 
     def test_appointments_filtered_for_specified_bso(self):
-        clinic1 = ClinicFactory(bso_code="MDB", code="BU001")
+        clinic1 = ClinicFactory(bso_code="MDB", name="Breast Care Unit", code="BU001")
         clinic2 = ClinicFactory(bso_code="WAT", code="BU002")
         AppointmentFactory.create_batch(size=2, clinic=clinic1)
         AppointmentFactory.create_batch(size=2, clinic=clinic2)
 
-        results = Helper.fetchall(
-            "reconciliation",
-            (
-                datetime.now(tz=ZONE_INFO).date(),
-                "MDB",
-            ),
-        )
-        assert len(results) == 1
-        assert results[0] == ("BU001", "S", "B", 2)
+        results = Helper.fetchall("reconciliation", ["1 week", "MDB"])
+        assert len(results) == 2
+        assert ResultRow(*results[0]).clinic == "Breast Care Unit (BU001)"