[DTOSS-11551] Add extract model

manage_breast_screening/notifications/management/commands/create_appointments.py

@@ -14,6 +14,7 @@
     Appointment,
     AppointmentStatusChoices,
     Clinic,
+    Extract,
 )
 from manage_breast_screening.notifications.services.blob_storage import BlobStorage
 
@@ -53,6 +54,8 @@ def handle(self, *args, **options):
 
                 data_frame = self.raw_data_to_data_frame(blob_content)
 
+                extract = self.create_extract(blob.name, blob_content)
+
                 for idx, row in data_frame.iterrows():
                     if self.is_not_holding_clinic(row):
                         clinic, clinic_created = self.find_or_create_clinic(row)
@@ -62,8 +65,26 @@ def handle(self, *args, **options):
                         appt, appt_created = self.update_or_create_appointment(
                             row, clinic
                         )
+
+                        extract.appointments.add(appt) if appt is not None else None
+
                 logger.info("Processed %s rows from %s", len(data_frame), blob.name)
 
+    def create_extract(self, filename: str, raw_data: str) -> Extract:
+        bso_code = filename.split("/")[1].split("_")[0]
+        type_id, extract_id, start_date, start_time, record_count = raw_data.split(
+            "\n"
+        )[0].split("|")
+        formatted_extract_id = int(extract_id.replace('"', "").replace("\r", ""))
+        formatted_record_count = int(record_count.replace('"', "").replace("\r", ""))
+
+        return Extract.objects.create(
+            sequence_number=formatted_extract_id,
+            bso_code=bso_code,
+            filename=filename,
+            record_count=formatted_record_count,
+        )
+
     def is_not_holding_clinic(self, row):
         return row.get("Holding Clinic") != "Y"
 

manage_breast_screening/notifications/migrations/0022_extract.py

@@ -0,0 +1,29 @@
+# Generated by Django 5.2.8 on 2025-11-26 16:24
+
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('notifications', '0021_alter_clinic_code_clinic_notificatio_code_55dbdb_idx_and_more'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Extract',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('filename', models.CharField(max_length=255)),
+                ('bso_code', models.CharField(max_length=255)),
+                ('sequence_number', models.IntegerField()),
+                ('record_count', models.IntegerField()),
+                ('appointments', models.ManyToManyField(blank=True, related_name='extracts', to='notifications.appointment')),
+            ],
+            options={
+                'constraints': [models.UniqueConstraint(fields=('bso_code', 'sequence_number'), name='unique_extract_code')],
+            },
+        ),
+    ]

manage_breast_screening/notifications/migrations/max_migration.txt

@@ -1 +1 @@
-0021_alter_clinic_code_clinic_notificatio_code_55dbdb_idx_and_more
+0022_extract

manage_breast_screening/notifications/models.py

@@ -216,3 +216,25 @@ class MessageStatus(models.Model):
     status_updated_at = models.DateTimeField(null=False)
     created_at = models.DateTimeField(null=False, auto_now_add=True)
     updated_at = models.DateTimeField(null=False, auto_now_add=True)
+
+
+class Extract(models.Model):
+    """
+    A model to store the extracted data from the message
+    """
+
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    appointments = models.ManyToManyField(
+        "notifications.Appointment", blank=True, related_name="extracts")
+    created_at = models.DateTimeField(auto_now_add=True)
+    filename = models.CharField(max_length=255, null=False)
+    bso_code = models.CharField(max_length=255, null=False)
+    sequence_number = models.IntegerField(null=False)
+    record_count = models.IntegerField(null=False)
+
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(
+                fields=["bso_code", "sequence_number"], name="unique_extract_code"
+            )
+        ]

manage_breast_screening/notifications/tests/fixtures/ABC_20241202091221_APPT_110.dat

@@ -0,0 +1 @@
+wrong data format here

manage_breast_screening/notifications/tests/fixtures/ABC_20241202091321_APPT_107.dat

@@ -1,4 +1,4 @@
-"NBSSAPPT_HDR"|"00000013"|"20250128"|"170922"|"000003"
+"NBSSAPPT_HDR"|"00000014"|"20250128"|"170922"|"000003"
 "NBSSAPPT_FLDS"|"Sequence"|"BSO"|"Action"|"Clinic Code"|"Holding Clinic"|"Status"|"Attended Not Scr"|"Appointment ID"|"NHS Num"|"Episode Type"|"Episode Start"|"Batch ID"|"Screen or Asses"|"Screen Appt num"|"Booked By"|"Cancelled By"|"Appt Date"|"Appt Time"|"Location"|"Clinic Name"|"Clinic Name (Let)"|"Clinic Address 1"|"Clinic Address 2"|"Clinic Address 3"|"Clinic Address 4"|"Clinic Address 5"|"Postcode"|"Action Timestamp"
 "NBSSAPPT_DATA"|"000001"|"KMK"|"C"|"BU011"|"N"|"C"|"N"|"BU011-67278-RA1-DN-Y1111-1"|"9449305552"|"F"|"20250128"|"KMK001326"|"S"|"1"|"H"|"C"|"20250314"|"1345"|"MKGH"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"MILTON KEYNES HOSPITAL"|"STANDING WAY"|"MILTON KEYNES"|"MK6 5LD"|"MK6 5LD"|"20250128-175555"
 "NBSSAPPT_END"|"00000013"|"20250128"|"17:09:22"|"000003"

manage_breast_screening/notifications/tests/fixtures/ABC_20241202091421_APPT_108.dat

@@ -1,4 +1,4 @@
-"NBSSAPPT_HDR"|"00000013"|"20250128"|"170922"|"000003"
+"NBSSAPPT_HDR"|"00000015"|"20250128"|"170922"|"000003"
 "NBSSAPPT_FLDS"|"Sequence"|"BSO"|"Action"|"Clinic Code"|"Holding Clinic"|"Status"|"Attended Not Scr"|"Appointment ID"|"NHS Num"|"Episode Type"|"Episode Start"|"Batch ID"|"Screen or Asses"|"Screen Appt num"|"Booked By"|"Cancelled By"|"Appt Date"|"Appt Time"|"Location"|"Clinic Name"|"Clinic Name (Let)"|"Clinic Address 1"|"Clinic Address 2"|"Clinic Address 3"|"Clinic Address 4"|"Clinic Address 5"|"Postcode"|"Action Timestamp"
 "NBSSAPPT_DATA"|"000001"|"KMK"|"C"|"BU003"|"N"|"C"|"N"|"BU003-67215-RA1-DN-Z2222-1"|"9449304424"|"S"|"20250102"|"KMKS02441"|"S"|""|"C"|"H"|"20250110"|"0845"|"MKGH"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"MILTON KEYNES HOSPITAL"|"STANDING WAY"|"MILTON KEYNES"|"MK6 5LD"|"MK6 5LD"|"20250123-121433"
 "NBSSAPPT_DATA"|"000002"|"KMK"|"B"|"BU011"|"Y"|"B"|"N"|"BU011-67278-RA1-DN-X0000-4"|"9449306625"|"F"|"20250128"|"KMK001326"|"S"|"1"|"H"|""|"20250314"|"1445"|"MKGH"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"MILTON KEYNES HOSPITAL"|"STANDING WAY"|"MILTON KEYNES"|"MK6 5LD"|"MK6 5LD"|"20250128-154003"

manage_breast_screening/notifications/tests/fixtures/ABC_20241202091521_APPT_109.dat

@@ -1,4 +1,4 @@
-"NBSSAPPT_HDR"|"00000013"|"20250128"|"170922"|"000003"
+"NBSSAPPT_HDR"|"00000016"|"20250128"|"170922"|"000003"
 "NBSSAPPT_FLDS"|"Sequence"|"BSO"|"Action"|"Clinic Code"|"Holding Clinic"|"Status"|"Attended Not Scr"|"Appointment ID"|"NHS Num"|"Episode Type"|"Episode Start"|"Batch ID"|"Screen or Asses"|"Screen Appt num"|"Booked By"|"Cancelled By"|"Appt Date"|"Appt Time"|"Location"|"Clinic Name"|"Clinic Name (Let)"|"Clinic Address 1"|"Clinic Address 2"|"Clinic Address 3"|"Clinic Address 4"|"Clinic Address 5"|"Postcode"|"Action Timestamp"
 "NBSSAPPT_DATA"|"000007"|"KMK"|"B"|"BU011"|"N"|"A"|"N"|"BU011-67278-RA1-DN-Y1111-1"|"9449305552"|"S"|"20250128"|"KMK001326"|"S"|"1"|"H"|""|"20250314"|"1345"|"MKGH"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"MILTON KEYNES HOSPITAL"|"STANDING WAY"|"MILTON KEYNES"|"MK6 5LD"|"MK6 5LD"|"20250128-154003"
 "NBSSAPPT_DATA"|"000008"|"KMK"|"B"|"BU011"|"N"|"D"|""|"BU011-67278-RA1-DN-X0000-1"|"9449306621"|"F"|"20250128"|"KMK001326"|"S"|"1"|"H"|""|"20250314"|"1445"|"MKGH"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"MILTON KEYNES HOSPITAL"|"STANDING WAY"|"MILTON KEYNES"|"MK6 5LD"|"MK6 5LD"|"20250128-154004"

manage_breast_screening/notifications/tests/management/commands/test_create_appointments.py

@@ -1,7 +1,7 @@
 import os
 from contextlib import contextmanager
 from datetime import datetime, timezone
-from unittest.mock import Mock, PropertyMock, patch
+from unittest.mock import Mock, patch
 
 import pytest
 from azure.storage.blob import BlobProperties
@@ -10,7 +10,12 @@
 from manage_breast_screening.notifications.management.commands.create_appointments import (
     Command,
 )
-from manage_breast_screening.notifications.models import ZONE_INFO, Appointment, Clinic
+from manage_breast_screening.notifications.models import (
+    ZONE_INFO,
+    Appointment,
+    Clinic,
+    Extract,
+)
 from manage_breast_screening.notifications.tests.factories import (
     AppointmentFactory,
     ClinicFactory,
@@ -20,6 +25,7 @@
 UPDATED_APPOINTMENT_FILE = "ABC_20241202091321_APPT_107.dat"
 HOLDING_CLINIC_APPOINTMENT_FILE = "ABC_20241202091421_APPT_108.dat"
 COMPLETED_APPOINTMENT_FILE = "ABC_20241202091521_APPT_109.dat"
+WRONG_DATA_FILE = "ABC_20241202091221_APPT_110.dat"
 
 
 def fixture_file_path(filename):
@@ -44,7 +50,7 @@ def stored_blob_data(prefix_dir: str, filenames: list[str]):
         mock_blob_contents = []
         for filename in filenames:
             mock_blob = Mock(spec=BlobProperties)
-            mock_blob.name = PropertyMock(return_value=f"{prefix_dir}/{filename}")
+            mock_blob.name = f"{prefix_dir}/{filename}"
             mock_blobs.append(mock_blob)
             mock_blob_contents.append(open(fixture_file_path(filename)).read())
 
@@ -118,6 +124,9 @@ def test_handle_creates_records(self):
 
         assert appointments[1].assessment is True
 
+        assert Extract.objects.count() == 1
+        assert Extract.objects.first().appointments.count() == 2
+
     def test_handles_holding_clinics(self):
         """Test does not create appointments for valid NBSS data marked as a Holding Clinic"""
         today_dirname = datetime.today().strftime("%Y-%m-%d")
@@ -318,3 +327,67 @@ def test_calls_command_handler(
             Command().handle(**{"date_str": "2000-01-01"})
 
         mock_command_handler.assert_called_with("CreateAppointments")
+
+    def test_create_extract_and_cancel(self):
+        """Test Extract creation for new booked appointments in NBSS data, stored in Azure storage blob"""
+        today_dirname = datetime.now().strftime("%Y-%m-%d")
+
+        with stored_blob_data(today_dirname, [VALID_DATA_FILE]):
+            Command().handle(**{"date_str": today_dirname})
+
+        assert Extract.objects.count() == 1
+        first_extract = Extract.objects.all()[0]
+        assert first_extract.appointments.count() == 2
+        assert first_extract.sequence_number == 13
+        assert first_extract.bso_code == "ABC"
+        assert first_extract.filename == f"{today_dirname}/{VALID_DATA_FILE}"
+        assert first_extract.record_count == 3
+
+        appointment_to_update = Appointment.objects.filter(
+            nbss_id="BU011-67278-RA1-DN-Y1111-1"
+        ).first()
+        assert appointment_to_update.extracts.count() == 1
+
+        with stored_blob_data(today_dirname, [UPDATED_APPOINTMENT_FILE]):
+            Command().handle(**{"date_str": today_dirname})
+
+        assert Extract.objects.count() == 2
+
+        assert appointment_to_update.extracts.count() == 2
+
+    @pytest.mark.django_db(transaction=True)
+    def test_errors_when_same_extract(self):
+        today_dirname = datetime.now().strftime("%Y-%m-%d")
+
+        with stored_blob_data(today_dirname, [VALID_DATA_FILE]):
+            Command().handle(**{"date_str": today_dirname})
+
+        assert Extract.objects.count() == 1
+        assert Appointment.objects.count() == 2
+
+        with stored_blob_data(today_dirname, [VALID_DATA_FILE]):
+            with pytest.raises(CommandError):
+                Command().handle(**{"date_str": today_dirname})
+
+        assert Extract.objects.count() == 1
+        assert Appointment.objects.count() == 2
+
+    @pytest.mark.django_db(transaction=True)
+    def test_errors_with_wrong_format_filename(self):
+        today_dirname = datetime.now().strftime("%Y-%m-%d")
+
+        with stored_blob_data(today_dirname, ["wrongfilename.dat"]):
+            with pytest.raises(CommandError):
+                Command().handle(**{"date_str": today_dirname})
+
+        assert Extract.objects.count() == 0
+
+    @pytest.mark.django_db(transaction=True)
+    def test_errors_with_wrong_format_data(self):
+        today_dirname = datetime.now().strftime("%Y-%m-%d")
+
+        with stored_blob_data(today_dirname, [WRONG_DATA_FILE]):
+            with pytest.raises(CommandError):
+                Command().handle(**{"date_str": today_dirname})
+
+        assert Extract.objects.count() == 0