add extract model and populate in create_appointments

Author: cameronhargreaves1-nhs

manage_breast_screening/notifications/management/commands/create_appointments.py

@@ -14,6 +14,7 @@
     Appointment,
     AppointmentStatusChoices,
     Clinic,
+    Extract,
 )
 from manage_breast_screening.notifications.services.blob_storage import BlobStorage
 
@@ -52,6 +53,11 @@ def handle(self, *args, **options):
                 ).readall()
 
                 data_frame = self.raw_data_to_data_frame(blob_content)
+                
+                extract = Extract.objects.create(created_at = datetime.now(tz=ZONE_INFO),
+                                                 bso_code = data_frame.iloc[0]["BSO"],
+                                                 filename = blob.name,
+                                                 sequence_number = data_frame.iloc[0]["Sequence"])
 
                 for idx, row in data_frame.iterrows():
                     if self.is_not_holding_clinic(row):
@@ -60,7 +66,7 @@ def handle(self, *args, **options):
                             logger.info("%s created", clinic)
 
                         appt, appt_created = self.update_or_create_appointment(
-                            row, clinic
+                            row, clinic, extract
                         )
                 logger.info("Processed %s rows from %s", len(data_frame), blob.name)
             logger.info("Create Appointments command finished successfully")
@@ -99,7 +105,7 @@ def find_or_create_clinic(self, row: pandas.Series) -> tuple[Clinic, bool]:
         )
 
     def update_or_create_appointment(
-        self, row: pandas.Series, clinic: Clinic
+        self, row: pandas.Series, clinic: Clinic, extract: Extract
     ) -> tuple[Appointment | None, bool]:
         appointment = Appointment.objects.filter(nbss_id=row["Appointment ID"]).first()
 
@@ -127,6 +133,7 @@ def update_or_create_appointment(
                     == "A"
                 ),
             )
+            extract.appointments.add(new_appointment)
             logger.info("%s created", new_appointment)
             return (new_appointment, True)
         elif self.is_cancelling_existing_appointment(row, appointment):
@@ -136,6 +143,7 @@ def update_or_create_appointment(
                 row["Action Timestamp"]
             )
             appointment.save()
+            extract.appointments.add(appointment)
             logger.info("%s cancelled", appointment)
         elif self.is_completed_appointment(row, appointment):
             appointment.status = row["Status"]
@@ -144,6 +152,7 @@ def update_or_create_appointment(
                 row["Action Timestamp"]
             )
             appointment.save()
+            extract.appointments.add(appointment)
             logger.info("%s marked completed (%s)", appointment, row.get("Status"))
         elif appointment is None:
             logger.info(

manage_breast_screening/notifications/migrations/0022_extract.py

@@ -0,0 +1,25 @@
+# Generated by Django 5.2.7 on 2025-11-19 08:28
+
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('notifications', '0021_alter_clinic_code_clinic_notificatio_code_55dbdb_idx_and_more'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Extract',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('filename', models.CharField(max_length=255)),
+                ('bso_code', models.CharField(max_length=255)),
+                ('sequence_number', models.IntegerField()),
+                ('appointments', models.ManyToManyField(blank=True, to='notifications.appointment')),
+            ],
+        ),
+    ]

manage_breast_screening/notifications/models.py

@@ -216,3 +216,21 @@ class MessageStatus(models.Model):
     status_updated_at = models.DateTimeField(null=False)
     created_at = models.DateTimeField(null=False, auto_now_add=True)
     updated_at = models.DateTimeField(null=False, auto_now_add=True)
+
+
+class Extract(models.Model):
+    """
+    A model to store the extracted data from the message
+    """
+
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    appointments = models.ManyToManyField("notifications.Appointment", blank=True)
+    created_at = models.DateTimeField(auto_now_add=True)
+    filename = models.CharField(max_length=255, null=False)
+    bso_code = models.CharField(max_length=255, null=False)
+    sequence_number = models.IntegerField(null=False)
+    constraints = [
+            models.UniqueConstraint(
+                fields=["bso_code", "sequence_number"], name="unique_code_bso_code"
+            )
+    ]

manage_breast_screening/notifications/tests/fixtures/ABC_20251118150721_APPT_101.dat

@@ -0,0 +1,4 @@
+"NBSSAPPT_HDR"|"00000013"|"20250128"|"170922"|"000003"
+"NBSSAPPT_FLDS"|"Sequence"|"BSO"|"Action"|"Clinic Code"|"Holding Clinic"|"Status"|"Attended Not Scr"|"Appointment ID"|"NHS Num"|"Epsiode Type"|"Episode Start"|"BatchID"|"Screen or Asses"|"Screen Appt num"|"Booked By"|"Cancelled By"|"Appt Date"|"Appt Time"|"Location"|"Clinic Name"|"Clinic Name (Let)"|"Clinic Address 1"|"Clinic Address 2"|"Clinic Address 3"|"Clinic Address 4"|"Clinic Address 5"|"Postcode"|"Action Timestamp"
+"NBSSAPPT_DATA"|"000001"|"KMK"|"B"|"BU003"|"N"|"B"|"N"|"BU003-67215-RA1-DN-Z2222-1"|"9449304424"|"S"|"20250102"|"KMKS02441"|"S"|""|"C"|"H"|"20250110"|"0845"|"MKGH"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"MILTON KEYNES HOSPITAL"|"STANDING WAY"|"MILTON KEYNES"|"MK6 5LD"|"MK6 5LD"|"20250123-121433"
+"NBSSAPPT_END"|"00000013"|"20250128"|"17:09:22"|"000003"

manage_breast_screening/notifications/tests/fixtures/ABC_20251118150721_APPT_102.dat

@@ -0,0 +1,4 @@
+"NBSSAPPT_HDR"|"00000013"|"20250128"|"170922"|"000003"
+"NBSSAPPT_FLDS"|"Sequence"|"BSO"|"Action"|"Clinic Code"|"Holding Clinic"|"Status"|"Attended Not Scr"|"Appointment ID"|"NHS Num"|"Epsiode Type"|"Episode Start"|"BatchID"|"Screen or Asses"|"Screen Appt num"|"Booked By"|"Cancelled By"|"Appt Date"|"Appt Time"|"Location"|"Clinic Name"|"Clinic Name (Let)"|"Clinic Address 1"|"Clinic Address 2"|"Clinic Address 3"|"Clinic Address 4"|"Clinic Address 5"|"Postcode"|"Action Timestamp"
+"NBSSAPPT_DATA"|"000001"|"KMK"|"C"|"BU003"|"N"|"C"|"N"|"BU003-67215-RA1-DN-Z2222-1"|"9449304424"|"S"|"20250102"|"KMKS02441"|"S"|""|"C"|"H"|"20250110"|"0845"|"MKGH"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"MILTON KEYNES HOSPITAL"|"STANDING WAY"|"MILTON KEYNES"|"MK6 5LD"|"MK6 5LD"|"20250123-121433"
+"NBSSAPPT_END"|"00000013"|"20250128"|"17:09:22"|"000003"

manage_breast_screening/notifications/tests/management/commands/test_create_appointments.py

@@ -1,7 +1,7 @@
 import os
 from contextlib import contextmanager
 from datetime import datetime, timezone
-from unittest.mock import Mock, PropertyMock, patch
+from unittest.mock import Mock, patch
 
 import pytest
 from azure.storage.blob import BlobProperties
@@ -10,7 +10,12 @@
 from manage_breast_screening.notifications.management.commands.create_appointments import (
     Command,
 )
-from manage_breast_screening.notifications.models import ZONE_INFO, Appointment, Clinic
+from manage_breast_screening.notifications.models import (
+    ZONE_INFO,
+    Appointment,
+    Clinic,
+    Extract,
+)
 from manage_breast_screening.notifications.tests.factories import (
     AppointmentFactory,
     ClinicFactory,
@@ -20,6 +25,8 @@
 UPDATED_APPOINTMENT_FILE = "ABC_20241202091321_APPT_107.dat"
 HOLDING_CLINIC_APPOINTMENT_FILE = "ABC_20241202091421_APPT_108.dat"
 COMPLETED_APPOINTMENT_FILE = "ABC_20241202091521_APPT_109.dat"
+SINGLE_APPOINTMENT_FILE = "ABC_20251118150721_APPT_101.dat"
+CANCELLED_SINGLE_APPOINTMENT_FILE = "ABC_20251118150721_APPT_102.dat"
 
 
 def fixture_file_path(filename):
@@ -44,7 +51,7 @@ def stored_blob_data(prefix_dir: str, filenames: list[str]):
         mock_blob_contents = []
         for filename in filenames:
             mock_blob = Mock(spec=BlobProperties)
-            mock_blob.name = PropertyMock(return_value=f"{prefix_dir}/{filename}")
+            mock_blob.name = f"{prefix_dir}/{filename}"
             mock_blobs.append(mock_blob)
             mock_blob_contents.append(open(fixture_file_path(filename)).read())
 
@@ -118,6 +125,9 @@ def test_handle_creates_records(self):
 
         assert appointments[1].assessment is True
 
+        assert Extract.objects.count() == 1
+        assert Extract.objects.first().appointments.count() == 2
+
     def test_handles_holding_clinics(self):
         """Test does not create appointments for valid NBSS data marked as a Holding Clinic"""
         today_dirname = datetime.today().strftime("%Y-%m-%d")
@@ -196,7 +206,7 @@ def test_only_updates_cancelled_appointments(self):
             ),
             status="C",
         )
-
+        
         nbss_id_cancelled = "BU003-67215-RA1-DN-Z2222-1"
         existing_appt_to_cancel = AppointmentFactory(
             nbss_id=nbss_id_cancelled,
@@ -320,3 +330,22 @@ def test_calls_insights_logger_if_exception_raised(
                 Command().handle(**{"date_str": "Noooo!"})
 
         mock_insights_logger.assert_called_with("CreateAppointmentsError: Error!")
+
+    def test_create_extract_and_cancel(self):
+        """Test Extract creation for new booked appointments in NBSS data, stored in Azure storage blob"""
+        today_dirname = datetime.now().strftime("%Y-%m-%d")
+
+        with stored_blob_data(today_dirname, [SINGLE_APPOINTMENT_FILE]):
+            Command().handle(**{"date_str": today_dirname})
+            
+        assert Extract.objects.count() == 1
+        assert Extract.objects.first().appointments.count() == 1
+        assert Extract.objects.first().appointments
+        
+        
+        with stored_blob_data(today_dirname, [CANCELLED_SINGLE_APPOINTMENT_FILE]):
+            Command().handle(**{"date_str": today_dirname})
+            
+        assert Extract.objects.count() == 2
+        
+        assert Extract.objects.all()[0].appointments.get().id == Extract.objects.all()[1].appointments.get().id