wip add extract info to appointments on creation

to do:
tidy up file name extraction
tidy up different .dat files and testing scenarios

Author: Harriethw

manage_breast_screening/notifications/management/commands/create_appointments.py

@@ -14,6 +14,7 @@
     Appointment,
     AppointmentStatusChoices,
     Clinic,
+    Extract,
 )
 from manage_breast_screening.notifications.services.blob_storage import BlobStorage
 
@@ -53,6 +54,15 @@ def handle(self, *args, **options):
 
                 data_frame = self.raw_data_to_data_frame(blob_content)
 
+                file_headers = self.get_file_header(blob_content)
+
+                extract = Extract.objects.create(
+                    sequence_number=int(file_headers[1].strip()),
+                    filename=blob.name,
+                    bso_code=data_frame.iloc[0]["BSO"],
+                    record_count=file_headers[4],
+                )
+
                 for idx, row in data_frame.iterrows():
                     if self.is_not_holding_clinic(row):
                         clinic, clinic_created = self.find_or_create_clinic(row)
@@ -62,6 +72,9 @@ def handle(self, *args, **options):
                         appt, appt_created = self.update_or_create_appointment(
                             row, clinic
                         )
+
+                        extract.appointments.add(appt) if appt is not None else None
+
                 logger.info("Processed %s rows from %s", len(data_frame), blob.name)
 
     def is_not_holding_clinic(self, row):
@@ -79,6 +92,9 @@ def raw_data_to_data_frame(self, raw_data: str) -> pandas.DataFrame:
             skipfooter=1,
         )
 
+    def get_file_header(self, raw_data: str) -> pandas.Series:
+        return raw_data.split("\n")[0].replace('"', "").split("|")
+
     def find_or_create_clinic(self, row: pandas.Series) -> tuple[Clinic, bool]:
         return Clinic.objects.get_or_create(
             bso_code=row["BSO"],

manage_breast_screening/notifications/tests/fixtures/ABC_20241202091321_APPT_107.dat

@@ -1,4 +1,4 @@
-"NBSSAPPT_HDR"|"00000013"|"20250128"|"170922"|"000003"
+"NBSSAPPT_HDR"|"00000014"|"20250128"|"170922"|"000003"
 "NBSSAPPT_FLDS"|"Sequence"|"BSO"|"Action"|"Clinic Code"|"Holding Clinic"|"Status"|"Attended Not Scr"|"Appointment ID"|"NHS Num"|"Episode Type"|"Episode Start"|"Batch ID"|"Screen or Asses"|"Screen Appt num"|"Booked By"|"Cancelled By"|"Appt Date"|"Appt Time"|"Location"|"Clinic Name"|"Clinic Name (Let)"|"Clinic Address 1"|"Clinic Address 2"|"Clinic Address 3"|"Clinic Address 4"|"Clinic Address 5"|"Postcode"|"Action Timestamp"
 "NBSSAPPT_DATA"|"000001"|"KMK"|"C"|"BU011"|"N"|"C"|"N"|"BU011-67278-RA1-DN-Y1111-1"|"9449305552"|"F"|"20250128"|"KMK001326"|"S"|"1"|"H"|"C"|"20250314"|"1345"|"MKGH"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"MILTON KEYNES HOSPITAL"|"STANDING WAY"|"MILTON KEYNES"|"MK6 5LD"|"MK6 5LD"|"20250128-175555"
 "NBSSAPPT_END"|"00000013"|"20250128"|"17:09:22"|"000003"

manage_breast_screening/notifications/tests/fixtures/ABC_20241202091521_APPT_109.dat

@@ -1,4 +1,4 @@
-"NBSSAPPT_HDR"|"00000013"|"20250128"|"170922"|"000003"
+"NBSSAPPT_HDR"|"00000014"|"20250128"|"170922"|"000003"
 "NBSSAPPT_FLDS"|"Sequence"|"BSO"|"Action"|"Clinic Code"|"Holding Clinic"|"Status"|"Attended Not Scr"|"Appointment ID"|"NHS Num"|"Episode Type"|"Episode Start"|"Batch ID"|"Screen or Asses"|"Screen Appt num"|"Booked By"|"Cancelled By"|"Appt Date"|"Appt Time"|"Location"|"Clinic Name"|"Clinic Name (Let)"|"Clinic Address 1"|"Clinic Address 2"|"Clinic Address 3"|"Clinic Address 4"|"Clinic Address 5"|"Postcode"|"Action Timestamp"
 "NBSSAPPT_DATA"|"000007"|"KMK"|"B"|"BU011"|"N"|"A"|"N"|"BU011-67278-RA1-DN-Y1111-1"|"9449305552"|"S"|"20250128"|"KMK001326"|"S"|"1"|"H"|""|"20250314"|"1345"|"MKGH"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"MILTON KEYNES HOSPITAL"|"STANDING WAY"|"MILTON KEYNES"|"MK6 5LD"|"MK6 5LD"|"20250128-154003"
 "NBSSAPPT_DATA"|"000008"|"KMK"|"B"|"BU011"|"N"|"D"|""|"BU011-67278-RA1-DN-X0000-1"|"9449306621"|"F"|"20250128"|"KMK001326"|"S"|"1"|"H"|""|"20250314"|"1445"|"MKGH"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"MILTON KEYNES HOSPITAL"|"STANDING WAY"|"MILTON KEYNES"|"MK6 5LD"|"MK6 5LD"|"20250128-154004"

manage_breast_screening/notifications/tests/management/commands/test_create_appointments.py

@@ -1,7 +1,7 @@
 import os
 from contextlib import contextmanager
 from datetime import datetime, timezone
-from unittest.mock import Mock, PropertyMock, patch
+from unittest.mock import Mock, patch
 
 import pytest
 from azure.storage.blob import BlobProperties
@@ -10,7 +10,12 @@
 from manage_breast_screening.notifications.management.commands.create_appointments import (
     Command,
 )
-from manage_breast_screening.notifications.models import ZONE_INFO, Appointment, Clinic
+from manage_breast_screening.notifications.models import (
+    ZONE_INFO,
+    Appointment,
+    Clinic,
+    Extract,
+)
 from manage_breast_screening.notifications.tests.factories import (
     AppointmentFactory,
     ClinicFactory,
@@ -44,7 +49,7 @@ def stored_blob_data(prefix_dir: str, filenames: list[str]):
         mock_blob_contents = []
         for filename in filenames:
             mock_blob = Mock(spec=BlobProperties)
-            mock_blob.name = PropertyMock(return_value=f"{prefix_dir}/{filename}")
+            mock_blob.name = f"{prefix_dir}/{filename}"
             mock_blobs.append(mock_blob)
             mock_blob_contents.append(open(fixture_file_path(filename)).read())
 
@@ -118,6 +123,9 @@ def test_handle_creates_records(self):
 
         assert appointments[1].assessment is True
 
+        assert Extract.objects.count() == 1
+        assert Extract.objects.first().appointments.count() == 2
+
     def test_handles_holding_clinics(self):
         """Test does not create appointments for valid NBSS data marked as a Holding Clinic"""
         today_dirname = datetime.today().strftime("%Y-%m-%d")
@@ -318,3 +326,44 @@ def test_calls_command_handler(
             Command().handle(**{"date_str": "2000-01-01"})
 
         mock_command_handler.assert_called_with("CreateAppointments")
+
+    def test_create_extract_and_cancel(self):
+        """Test Extract creation for new booked appointments in NBSS data, stored in Azure storage blob"""
+        today_dirname = datetime.now().strftime("%Y-%m-%d")
+
+        with stored_blob_data(today_dirname, [VALID_DATA_FILE]):
+            Command().handle(**{"date_str": today_dirname})
+
+        assert Extract.objects.count() == 1
+        first_extract = Extract.objects.all()[0]
+        assert first_extract.appointments.count() == 2
+        assert first_extract.sequence_number == 13
+        assert first_extract.bso_code == "KMK"
+        assert first_extract.filename == f"{today_dirname}/{VALID_DATA_FILE}"
+        assert first_extract.record_count == 3
+
+        with stored_blob_data(today_dirname, [UPDATED_APPOINTMENT_FILE]):
+            Command().handle(**{"date_str": today_dirname})
+
+        assert Extract.objects.count() == 2
+        second_extract = Extract.objects.all()[1]
+
+        assert (
+            first_extract.appointments.first().id
+            == second_extract.appointments.get().id
+        )
+
+    @pytest.mark.django_db(transaction=True)
+    def test_errors_when_same_extract(self):
+        today_dirname = datetime.now().strftime("%Y-%m-%d")
+
+        with stored_blob_data(today_dirname, [VALID_DATA_FILE]):
+            Command().handle(**{"date_str": today_dirname})
+
+        assert Appointment.objects.count() == 2
+
+        with stored_blob_data(today_dirname, [VALID_DATA_FILE]):
+            with pytest.raises(CommandError):
+                Command().handle(**{"date_str": today_dirname})
+
+        assert Appointment.objects.count() == 2