add record count field to extract model

Author: cameronhargreaves1-nhs

manage_breast_screening/notifications/management/commands/create_appointments.py

@@ -54,10 +54,13 @@ def handle(self, *args, **options):
 
                 data_frame = self.raw_data_to_data_frame(blob_content)
 
-                extract = Extract.objects.create(sequence_number = self.get_data_frame_sequence_number(blob_content),
+                file_headers = self.get_file_header(blob_content)
+                
+                extract = Extract.objects.create(sequence_number = int(file_headers[1].strip()),
                                                  filename = blob.name,
-                                                 bso_code = data_frame.iloc[0]["BSO"])
-
+                                                 bso_code = data_frame.iloc[0]["BSO"],
+                                                 record_count = file_headers[4])
+                
                 for idx, row in data_frame.iterrows():
                     if self.is_not_holding_clinic(row):
                         clinic, clinic_created = self.find_or_create_clinic(row)
@@ -68,26 +71,13 @@ def handle(self, *args, **options):
                             row, clinic, extract
                         )
 
-                        extract.appointments.add(appt) if appt is not None else None
+                        extract.rows.add(appt) if appt is not None else None
 
                 logger.info("Processed %s rows from %s", len(data_frame), blob.name)
 
     def is_not_holding_clinic(self, row):
         return row.get("Holding Clinic") != "Y"
 
-    def get_data_frame_sequence_number(self, raw_data: str) -> str:
-        data_frame = pandas.read_table(
-            io.StringIO(raw_data),
-            dtype="str",
-            encoding="ASCII",
-            engine="python",
-            header=0,
-            keep_default_na=False,
-            sep="|",
-            skipfooter=1,
-        )
-        return data_frame.columns.values[1]
-
     def raw_data_to_data_frame(self, raw_data: str) -> pandas.DataFrame:
         return pandas.read_table(
             io.StringIO(raw_data),
@@ -99,6 +89,9 @@ def raw_data_to_data_frame(self, raw_data: str) -> pandas.DataFrame:
             sep="|",
             skipfooter=1,
         )
+        
+    def get_file_header(self, raw_data: str) -> pandas.Series:
+        return raw_data.split("\n")[0].replace('"', '').split('|')
 
     def find_or_create_clinic(self, row: pandas.Series) -> tuple[Clinic, bool]:
         return Clinic.objects.get_or_create(

manage_breast_screening/notifications/migrations/0022_extract.py

@@ -1,4 +1,4 @@
-# Generated by Django 5.2.7 on 2025-11-19 08:28
+# Generated by Django 5.2.8 on 2025-11-21 15:41
 
 import uuid
 from django.db import migrations, models
@@ -19,7 +19,8 @@ class Migration(migrations.Migration):
                 ('filename', models.CharField(max_length=255)),
                 ('bso_code', models.CharField(max_length=255)),
                 ('sequence_number', models.IntegerField()),
-                ('appointments', models.ManyToManyField(blank=True, to='notifications.appointment')),
+                ('record_count', models.IntegerField()),
+                ('rows', models.ManyToManyField(blank=True, to='notifications.appointment')),
             ],
         ),
     ]

manage_breast_screening/notifications/migrations/max_migration.txt

@@ -1 +1 @@
-0021_alter_clinic_code_clinic_notificatio_code_55dbdb_idx_and_more
+0022_extract

manage_breast_screening/notifications/models.py

@@ -224,11 +224,12 @@ class Extract(models.Model):
     """
 
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
-    appointments = models.ManyToManyField("notifications.Appointment", blank=True)
+    rows = models.ManyToManyField("notifications.Appointment", blank=True)
     created_at = models.DateTimeField(auto_now_add=True)
     filename = models.CharField(max_length=255, null=False)
     bso_code = models.CharField(max_length=255, null=False)
     sequence_number = models.IntegerField(null=False)
+    record_count = models.IntegerField(null=False)
     constraints = [
             models.UniqueConstraint(
                 fields=["bso_code", "sequence_number"], name="unique_code_bso_code"

manage_breast_screening/notifications/tests/fixtures/ABC_20251118150721_APPT_101.dat

@@ -1,4 +1,4 @@
-"NBSSAPPT_HDR"|"00000013"|"20250128"|"170922"|"000003"
+"NBSSAPPT_HDR"|"00000013"|"20250128"|"170922"|"000001"
 "NBSSAPPT_FLDS"|"Sequence"|"BSO"|"Action"|"Clinic Code"|"Holding Clinic"|"Status"|"Attended Not Scr"|"Appointment ID"|"NHS Num"|"Epsiode Type"|"Episode Start"|"BatchID"|"Screen or Asses"|"Screen Appt num"|"Booked By"|"Cancelled By"|"Appt Date"|"Appt Time"|"Location"|"Clinic Name"|"Clinic Name (Let)"|"Clinic Address 1"|"Clinic Address 2"|"Clinic Address 3"|"Clinic Address 4"|"Clinic Address 5"|"Postcode"|"Action Timestamp"
 "NBSSAPPT_DATA"|"000001"|"KMK"|"B"|"BU003"|"N"|"B"|"N"|"BU003-67215-RA1-DN-Z2222-1"|"9449304424"|"S"|"20250102"|"KMKS02441"|"S"|""|"C"|"H"|"20250110"|"0845"|"MKGH"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"MILTON KEYNES HOSPITAL"|"STANDING WAY"|"MILTON KEYNES"|"MK6 5LD"|"MK6 5LD"|"20250123-121433"
-"NBSSAPPT_END"|"00000013"|"20250128"|"17:09:22"|"000003"
+"NBSSAPPT_END"|"00000013"|"20250128"|"17:09:22"|"000001"

manage_breast_screening/notifications/tests/fixtures/ABC_20251118150721_APPT_102.dat

@@ -1,4 +1,4 @@
-"NBSSAPPT_HDR"|"00000013"|"20250128"|"170922"|"000003"
+"NBSSAPPT_HDR"|"00000013"|"20250128"|"170922"|"000001"
 "NBSSAPPT_FLDS"|"Sequence"|"BSO"|"Action"|"Clinic Code"|"Holding Clinic"|"Status"|"Attended Not Scr"|"Appointment ID"|"NHS Num"|"Epsiode Type"|"Episode Start"|"BatchID"|"Screen or Asses"|"Screen Appt num"|"Booked By"|"Cancelled By"|"Appt Date"|"Appt Time"|"Location"|"Clinic Name"|"Clinic Name (Let)"|"Clinic Address 1"|"Clinic Address 2"|"Clinic Address 3"|"Clinic Address 4"|"Clinic Address 5"|"Postcode"|"Action Timestamp"
 "NBSSAPPT_DATA"|"000001"|"KMK"|"C"|"BU003"|"N"|"C"|"N"|"BU003-67215-RA1-DN-Z2222-1"|"9449304424"|"S"|"20250102"|"KMKS02441"|"S"|""|"C"|"H"|"20250110"|"0845"|"MKGH"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"MILTON KEYNES HOSPITAL"|"STANDING WAY"|"MILTON KEYNES"|"MK6 5LD"|"MK6 5LD"|"20250123-121433"
-"NBSSAPPT_END"|"00000013"|"20250128"|"17:09:22"|"000003"
+"NBSSAPPT_END"|"00000013"|"20250128"|"17:09:22"|"000001"

manage_breast_screening/notifications/tests/fixtures/ABC_20251118150721_APPT_103.dat

@@ -1,4 +1,4 @@
-"NBSSAPPT_HDR"|"00000013"|"20250128"|"170922"|"000003"
+"NBSSAPPT_HDR"|"00000013"|"20250128"|"170922"|"000001"
 "NBSSAPPT_FLDS"|"Sequence"|"BSO"|"Action"|"Clinic Code"|"Holding Clinic"|"Status"|"Attended Not Scr"|"Appointment ID"|"NHS Num"|"Epsiode Type"|"Episode Start"|"BatchID"|"Screen or Asses"|"Screen Appt num"|"Booked By"|"Cancelled By"|"Appt Date"|"Appt Time"|"Location"|"Clinic Name"|"Clinic Name (Let)"|"Clinic Address 1"|"Clinic Address 2"|"Clinic Address 3"|"Clinic Address 4"|"Clinic Address 5"|"Postcode"|"Action Timestamp"
 "NBSSAPPT_DATA"|"000001"|"KMK"|"C"|"BU003"|"N"|"D"|"N"|"BU003-67215-RA1-DN-Z2222-1"|"9449304424"|"S"|"20250102"|"KMKS02441"|"S"|""|"C"|"H"|"20250110"|"0845"|"MKGH"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"BREAST CARE UNIT"|"MILTON KEYNES HOSPITAL"|"STANDING WAY"|"MILTON KEYNES"|"MK6 5LD"|"MK6 5LD"|"20250123-121433"
-"NBSSAPPT_END"|"00000013"|"20250128"|"17:09:22"|"000003"
+"NBSSAPPT_END"|"00000013"|"20250128"|"17:09:22"|"000001"

manage_breast_screening/notifications/tests/management/commands/test_create_appointments.py

@@ -7,8 +7,6 @@
 from azure.storage.blob import BlobProperties
 from django.core.management.base import CommandError
 
-from psycopg.errors import NotNullViolation
-
 from manage_breast_screening.notifications.management.commands.create_appointments import (
     Command,
 )
@@ -129,7 +127,7 @@ def test_handle_creates_records(self):
         assert appointments[1].assessment is True
 
         assert Extract.objects.count() == 1
-        assert Extract.objects.first().appointments.count() == 2
+        assert Extract.objects.first().rows.count() == 2
 
     def test_handles_holding_clinics(self):
         """Test does not create appointments for valid NBSS data marked as a Holding Clinic"""
@@ -340,22 +338,23 @@ def test_create_extract_and_cancel(self):
             Command().handle(**{"date_str": today_dirname})
 
         assert Extract.objects.count() == 1
-        assert Extract.objects.first().appointments.count() == 1
-        assert Extract.objects.first().appointments
+        assert Extract.objects.first().rows.count() == 1
+        assert Extract.objects.first().rows
         assert Extract.objects.first().sequence_number == 13
         assert Extract.objects.first().bso_code == "KMK"
         assert Extract.objects.first().filename == f"{today_dirname}/ABC_20251118150721_APPT_101.dat"
+        assert Extract.objects.first().record_count == 1
 
         with stored_blob_data(today_dirname, [CANCELLED_SINGLE_APPOINTMENT_FILE]):
             Command().handle(**{"date_str": today_dirname})
 
         assert Extract.objects.count() == 2
 
-        assert Extract.objects.all()[0].appointments.get().id == Extract.objects.all()[1].appointments.get().id
+        assert Extract.objects.all()[0].rows.get().id == Extract.objects.all()[1].rows.get().id
 
         with stored_blob_data(today_dirname, [COMPLETED_SINGLE_APPOINTMENT_FILE]):
             Command().handle(**{"date_str": today_dirname})
 
         assert Extract.objects.count() == 3
 
-        assert Extract.objects.all()[0].appointments.get().id == Extract.objects.all()[2].appointments.get().id
+        assert Extract.objects.all()[0].rows.get().id == Extract.objects.all()[2].rows.get().id