diff --git a/manage_breast_screening/notifications/management/commands/create_appointments.py b/manage_breast_screening/notifications/management/commands/create_appointments.py index 00eae4ac4..315d6981c 100644 --- a/manage_breast_screening/notifications/management/commands/create_appointments.py +++ b/manage_breast_screening/notifications/management/commands/create_appointments.py @@ -14,6 +14,7 @@ Appointment, AppointmentStatusChoices, Clinic, + Extract, ) from manage_breast_screening.notifications.services.blob_storage import BlobStorage @@ -53,6 +54,8 @@ def handle(self, *args, **options): data_frame = self.raw_data_to_data_frame(blob_content) + extract = self.create_extract(blob.name, blob_content) + for idx, row in data_frame.iterrows(): if self.is_not_holding_clinic(row): clinic, clinic_created = self.find_or_create_clinic(row) @@ -62,8 +65,26 @@ def handle(self, *args, **options): appt, appt_created = self.update_or_create_appointment( row, clinic ) + + extract.appointments.add(appt) if appt is not None else None + logger.info("Processed %s rows from %s", len(data_frame), blob.name) + def create_extract(self, filename: str, raw_data: str) -> Extract: + bso_code = filename.split("/")[1].split("_")[0] + type_id, extract_id, start_date, start_time, record_count = raw_data.split( + "\n" + )[0].split("|") + formatted_extract_id = int(extract_id.replace('"', "").replace("\r", "")) + formatted_record_count = int(record_count.replace('"', "").replace("\r", "")) + + return Extract.objects.create( + sequence_number=formatted_extract_id, + bso_code=bso_code, + filename=filename, + record_count=formatted_record_count, + ) + def is_not_holding_clinic(self, row): return row.get("Holding Clinic") != "Y" diff --git a/manage_breast_screening/notifications/migrations/0022_extract.py b/manage_breast_screening/notifications/migrations/0022_extract.py new file mode 100644 index 000000000..864eb9a8a --- /dev/null +++ b/manage_breast_screening/notifications/migrations/0022_extract.py @@ -0,0 +1,29 @@ +# Generated by Django 5.2.8 on 2025-11-26 16:24 + +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('notifications', '0021_alter_clinic_code_clinic_notificatio_code_55dbdb_idx_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='Extract', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('filename', models.CharField(max_length=255)), + ('bso_code', models.CharField(max_length=255)), + ('sequence_number', models.IntegerField()), + ('record_count', models.IntegerField()), + ('appointments', models.ManyToManyField(blank=True, related_name='extracts', to='notifications.appointment')), + ], + options={ + 'constraints': [models.UniqueConstraint(fields=('bso_code', 'sequence_number'), name='unique_extract_code')], + }, + ), + ] diff --git a/manage_breast_screening/notifications/migrations/max_migration.txt b/manage_breast_screening/notifications/migrations/max_migration.txt index e0c21647f..e09b3183e 100644 --- a/manage_breast_screening/notifications/migrations/max_migration.txt +++ b/manage_breast_screening/notifications/migrations/max_migration.txt @@ -1 +1 @@ -0021_alter_clinic_code_clinic_notificatio_code_55dbdb_idx_and_more +0022_extract diff --git a/manage_breast_screening/notifications/models.py b/manage_breast_screening/notifications/models.py index b8e698bcd..2ddfe79f4 100644 --- a/manage_breast_screening/notifications/models.py +++ b/manage_breast_screening/notifications/models.py @@ -216,3 +216,25 @@ class MessageStatus(models.Model): status_updated_at = models.DateTimeField(null=False) created_at = models.DateTimeField(null=False, auto_now_add=True) updated_at = models.DateTimeField(null=False, auto_now_add=True) + + +class Extract(models.Model): + """ + A model to store the extracted data from the message + """ + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + appointments = models.ManyToManyField( + "notifications.Appointment", blank=True, related_name="extracts") + created_at = models.DateTimeField(auto_now_add=True) + filename = models.CharField(max_length=255, null=False) + bso_code = models.CharField(max_length=255, null=False) + sequence_number = models.IntegerField(null=False) + record_count = models.IntegerField(null=False) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["bso_code", "sequence_number"], name="unique_extract_code" + ) + ] diff --git a/manage_breast_screening/notifications/tests/fixtures/ABC_20241202091221_APPT_110.dat b/manage_breast_screening/notifications/tests/fixtures/ABC_20241202091221_APPT_110.dat new file mode 100644 index 000000000..8f5b70a29 --- /dev/null +++ b/manage_breast_screening/notifications/tests/fixtures/ABC_20241202091221_APPT_110.dat @@ -0,0 +1 @@ +wrong data format here diff --git a/manage_breast_screening/notifications/tests/fixtures/ABC_20241202091321_APPT_107.dat b/manage_breast_screening/notifications/tests/fixtures/ABC_20241202091321_APPT_107.dat index 7bb1a4352..f9c326f73 100644 --- a/manage_breast_screening/notifications/tests/fixtures/ABC_20241202091321_APPT_107.dat +++ b/manage_breast_screening/notifications/tests/fixtures/ABC_20241202091321_APPT_107.dat @@ -1,4 +1,4 @@ -"NBSSAPPT_HDR"|"00000013"|"20250128"|"170922"|"000003" +"NBSSAPPT_HDR"|"00000014"|"20250128"|"170922"|"000003" "NBSSAPPT_FLDS"|"Sequence"|"BSO"|"Action"|"Clinic Code"|"Holding Clinic"|"Status"|"Attended Not Scr"|"Appointment ID"|"NHS Num"|"Episode Type"|"Episode Start"|"Batch ID"|"Screen or Asses"|"Screen Appt num"|"Booked By"|"Cancelled By"|"Appt Date"|"Appt Time"|"Location"|"Clinic Name"|"Clinic Name (Let)"|"Clinic Address 1"|"Clinic Address 2"|"Clinic Address 3"|"Clinic Address 4"|"Clinic Address 5"|"Postcode"|"Action Timestamp" "NBSSAPPT_DATA"|"000001"|"KMK"|"C"|"BU011"|"N"|"C"|"N"|"BU011-67278-RA1-DN-Y1111-1"|"9449305552"|"F"|"20250128"|"KMK001326"|"S"|"1"|"H"|"C"|"20250314"|"1345"|"MKGH"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"MILTON KEYNES HOSPITAL"|"STANDING WAY"|"MILTON KEYNES"|"MK6 5LD"|"MK6 5LD"|"20250128-175555" "NBSSAPPT_END"|"00000013"|"20250128"|"17:09:22"|"000003" diff --git a/manage_breast_screening/notifications/tests/fixtures/ABC_20241202091421_APPT_108.dat b/manage_breast_screening/notifications/tests/fixtures/ABC_20241202091421_APPT_108.dat index 1ffa41f69..ff173cb98 100644 --- a/manage_breast_screening/notifications/tests/fixtures/ABC_20241202091421_APPT_108.dat +++ b/manage_breast_screening/notifications/tests/fixtures/ABC_20241202091421_APPT_108.dat @@ -1,4 +1,4 @@ -"NBSSAPPT_HDR"|"00000013"|"20250128"|"170922"|"000003" +"NBSSAPPT_HDR"|"00000015"|"20250128"|"170922"|"000003" "NBSSAPPT_FLDS"|"Sequence"|"BSO"|"Action"|"Clinic Code"|"Holding Clinic"|"Status"|"Attended Not Scr"|"Appointment ID"|"NHS Num"|"Episode Type"|"Episode Start"|"Batch ID"|"Screen or Asses"|"Screen Appt num"|"Booked By"|"Cancelled By"|"Appt Date"|"Appt Time"|"Location"|"Clinic Name"|"Clinic Name (Let)"|"Clinic Address 1"|"Clinic Address 2"|"Clinic Address 3"|"Clinic Address 4"|"Clinic Address 5"|"Postcode"|"Action Timestamp" "NBSSAPPT_DATA"|"000001"|"KMK"|"C"|"BU003"|"N"|"C"|"N"|"BU003-67215-RA1-DN-Z2222-1"|"9449304424"|"S"|"20250102"|"KMKS02441"|"S"|""|"C"|"H"|"20250110"|"0845"|"MKGH"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"MILTON KEYNES HOSPITAL"|"STANDING WAY"|"MILTON KEYNES"|"MK6 5LD"|"MK6 5LD"|"20250123-121433" "NBSSAPPT_DATA"|"000002"|"KMK"|"B"|"BU011"|"Y"|"B"|"N"|"BU011-67278-RA1-DN-X0000-4"|"9449306625"|"F"|"20250128"|"KMK001326"|"S"|"1"|"H"|""|"20250314"|"1445"|"MKGH"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"MILTON KEYNES HOSPITAL"|"STANDING WAY"|"MILTON KEYNES"|"MK6 5LD"|"MK6 5LD"|"20250128-154003" diff --git a/manage_breast_screening/notifications/tests/fixtures/ABC_20241202091521_APPT_109.dat b/manage_breast_screening/notifications/tests/fixtures/ABC_20241202091521_APPT_109.dat index 2b426fa96..4b11723a1 100644 --- a/manage_breast_screening/notifications/tests/fixtures/ABC_20241202091521_APPT_109.dat +++ b/manage_breast_screening/notifications/tests/fixtures/ABC_20241202091521_APPT_109.dat @@ -1,4 +1,4 @@ -"NBSSAPPT_HDR"|"00000013"|"20250128"|"170922"|"000003" +"NBSSAPPT_HDR"|"00000016"|"20250128"|"170922"|"000003" "NBSSAPPT_FLDS"|"Sequence"|"BSO"|"Action"|"Clinic Code"|"Holding Clinic"|"Status"|"Attended Not Scr"|"Appointment ID"|"NHS Num"|"Episode Type"|"Episode Start"|"Batch ID"|"Screen or Asses"|"Screen Appt num"|"Booked By"|"Cancelled By"|"Appt Date"|"Appt Time"|"Location"|"Clinic Name"|"Clinic Name (Let)"|"Clinic Address 1"|"Clinic Address 2"|"Clinic Address 3"|"Clinic Address 4"|"Clinic Address 5"|"Postcode"|"Action Timestamp" "NBSSAPPT_DATA"|"000007"|"KMK"|"B"|"BU011"|"N"|"A"|"N"|"BU011-67278-RA1-DN-Y1111-1"|"9449305552"|"S"|"20250128"|"KMK001326"|"S"|"1"|"H"|""|"20250314"|"1345"|"MKGH"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"MILTON KEYNES HOSPITAL"|"STANDING WAY"|"MILTON KEYNES"|"MK6 5LD"|"MK6 5LD"|"20250128-154003" "NBSSAPPT_DATA"|"000008"|"KMK"|"B"|"BU011"|"N"|"D"|""|"BU011-67278-RA1-DN-X0000-1"|"9449306621"|"F"|"20250128"|"KMK001326"|"S"|"1"|"H"|""|"20250314"|"1445"|"MKGH"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"MILTON KEYNES HOSPITAL"|"STANDING WAY"|"MILTON KEYNES"|"MK6 5LD"|"MK6 5LD"|"20250128-154004" diff --git a/manage_breast_screening/notifications/tests/fixtures/wrongfilename.dat b/manage_breast_screening/notifications/tests/fixtures/wrongfilename.dat new file mode 100644 index 000000000..e69de29bb diff --git a/manage_breast_screening/notifications/tests/management/commands/test_create_appointments.py b/manage_breast_screening/notifications/tests/management/commands/test_create_appointments.py index 10c883091..3fbf70b6d 100644 --- a/manage_breast_screening/notifications/tests/management/commands/test_create_appointments.py +++ b/manage_breast_screening/notifications/tests/management/commands/test_create_appointments.py @@ -1,7 +1,7 @@ import os from contextlib import contextmanager from datetime import datetime, timezone -from unittest.mock import Mock, PropertyMock, patch +from unittest.mock import Mock, patch import pytest from azure.storage.blob import BlobProperties @@ -10,7 +10,12 @@ from manage_breast_screening.notifications.management.commands.create_appointments import ( Command, ) -from manage_breast_screening.notifications.models import ZONE_INFO, Appointment, Clinic +from manage_breast_screening.notifications.models import ( + ZONE_INFO, + Appointment, + Clinic, + Extract, +) from manage_breast_screening.notifications.tests.factories import ( AppointmentFactory, ClinicFactory, @@ -20,6 +25,7 @@ UPDATED_APPOINTMENT_FILE = "ABC_20241202091321_APPT_107.dat" HOLDING_CLINIC_APPOINTMENT_FILE = "ABC_20241202091421_APPT_108.dat" COMPLETED_APPOINTMENT_FILE = "ABC_20241202091521_APPT_109.dat" +WRONG_DATA_FILE = "ABC_20241202091221_APPT_110.dat" def fixture_file_path(filename): @@ -44,7 +50,7 @@ def stored_blob_data(prefix_dir: str, filenames: list[str]): mock_blob_contents = [] for filename in filenames: mock_blob = Mock(spec=BlobProperties) - mock_blob.name = PropertyMock(return_value=f"{prefix_dir}/{filename}") + mock_blob.name = f"{prefix_dir}/{filename}" mock_blobs.append(mock_blob) mock_blob_contents.append(open(fixture_file_path(filename)).read()) @@ -118,6 +124,9 @@ def test_handle_creates_records(self): assert appointments[1].assessment is True + assert Extract.objects.count() == 1 + assert Extract.objects.first().appointments.count() == 2 + def test_handles_holding_clinics(self): """Test does not create appointments for valid NBSS data marked as a Holding Clinic""" today_dirname = datetime.today().strftime("%Y-%m-%d") @@ -318,3 +327,67 @@ def test_calls_command_handler( Command().handle(**{"date_str": "2000-01-01"}) mock_command_handler.assert_called_with("CreateAppointments") + + def test_create_extract_and_cancel(self): + """Test Extract creation for new booked appointments in NBSS data, stored in Azure storage blob""" + today_dirname = datetime.now().strftime("%Y-%m-%d") + + with stored_blob_data(today_dirname, [VALID_DATA_FILE]): + Command().handle(**{"date_str": today_dirname}) + + assert Extract.objects.count() == 1 + first_extract = Extract.objects.all()[0] + assert first_extract.appointments.count() == 2 + assert first_extract.sequence_number == 13 + assert first_extract.bso_code == "ABC" + assert first_extract.filename == f"{today_dirname}/{VALID_DATA_FILE}" + assert first_extract.record_count == 3 + + appointment_to_update = Appointment.objects.filter( + nbss_id="BU011-67278-RA1-DN-Y1111-1" + ).first() + assert appointment_to_update.extracts.count() == 1 + + with stored_blob_data(today_dirname, [UPDATED_APPOINTMENT_FILE]): + Command().handle(**{"date_str": today_dirname}) + + assert Extract.objects.count() == 2 + + assert appointment_to_update.extracts.count() == 2 + + @pytest.mark.django_db(transaction=True) + def test_errors_when_same_extract(self): + today_dirname = datetime.now().strftime("%Y-%m-%d") + + with stored_blob_data(today_dirname, [VALID_DATA_FILE]): + Command().handle(**{"date_str": today_dirname}) + + assert Extract.objects.count() == 1 + assert Appointment.objects.count() == 2 + + with stored_blob_data(today_dirname, [VALID_DATA_FILE]): + with pytest.raises(CommandError): + Command().handle(**{"date_str": today_dirname}) + + assert Extract.objects.count() == 1 + assert Appointment.objects.count() == 2 + + @pytest.mark.django_db(transaction=True) + def test_errors_with_wrong_format_filename(self): + today_dirname = datetime.now().strftime("%Y-%m-%d") + + with stored_blob_data(today_dirname, ["wrongfilename.dat"]): + with pytest.raises(CommandError): + Command().handle(**{"date_str": today_dirname}) + + assert Extract.objects.count() == 0 + + @pytest.mark.django_db(transaction=True) + def test_errors_with_wrong_format_data(self): + today_dirname = datetime.now().strftime("%Y-%m-%d") + + with stored_blob_data(today_dirname, [WRONG_DATA_FILE]): + with pytest.raises(CommandError): + Command().handle(**{"date_str": today_dirname}) + + assert Extract.objects.count() == 0