add Extract sequential check, send warning if fails

Author: cameronhargreaves1-nhs

manage_breast_screening/notifications/management/commands/create_appointments.py

@@ -23,6 +23,9 @@
 logger = getLogger(__name__)
 
 
+class ExtractValidationError(Exception):
+    pass
+
 class Command(BaseCommand):
     """
     Django Admin command which reads NBSS appointment data from Azure blob storage
@@ -56,9 +59,11 @@ def handle(self, *args, **options):
                 ).readall()
 
                 data_frame = self.raw_data_to_data_frame(blob_content)
+                
+                self.validate_extract(blob.name, blob_content)
 
                 extract = self.create_extract(blob.name, blob_content)
-
+                
                 for idx, row in data_frame.iterrows():
                     if self.is_not_holding_clinic(row):
                         clinic, clinic_created = self.find_or_create_clinic(row)
@@ -72,13 +77,33 @@ def handle(self, *args, **options):
                         extract.appointments.add(appt) if appt is not None else None
 
                 logger.info("Processed %s rows from %s", len(data_frame), blob.name)
+                
+    def validate_extract(self, filename: str, raw_data: str) -> None:
+        bso_code = filename.split("/")[1].split("_")[0]
+        type_id, extract_id, start_date, start_time, record_count = raw_data.split(
+            "\n"
+        )[0].split("|")
+        formatted_extract_id = int(extract_id.replace('"', "").replace("\r", ""))
+        
+        latest_extract = Extract.objects.filter(bso_code = bso_code).order_by("sequence_number").last()
+        
+        if latest_extract:            
+            if formatted_extract_id != (latest_extract.sequence_number + 1):
+                
+                log_msg = "Extract ID %s is not sequential to last extract ID %s." % (
+                    formatted_extract_id,
+                    latest_extract.sequence_number,
+                )
+                
+                raise ExtractValidationError(log_msg)
 
     def create_extract(self, filename: str, raw_data: str) -> Extract:
         bso_code = filename.split("/")[1].split("_")[0]
         type_id, extract_id, start_date, start_time, record_count = raw_data.split(
             "\n"
         )[0].split("|")
         formatted_extract_id = int(extract_id.replace('"', "").replace("\r", ""))
+        
         formatted_record_count = int(record_count.replace('"', "").replace("\r", ""))
 
         return Extract.objects.create(

manage_breast_screening/notifications/tests/management/commands/test_create_appointments.py

@@ -9,6 +9,7 @@
 
 from manage_breast_screening.notifications.management.commands.create_appointments import (
     Command,
+    ExtractValidationError,
 )
 from manage_breast_screening.notifications.models import (
     ZONE_INFO,
@@ -60,7 +61,6 @@ def stored_blob_data(prefix_dir: str, filenames: list[str]):
         )
         yield
 
-
 @pytest.mark.django_db
 class TestCreateAppointments:
     def test_handle_creates_records(self):
@@ -391,3 +391,54 @@ def test_errors_with_wrong_format_data(self):
                 Command().handle(**{"date_str": today_dirname})
 
         assert Extract.objects.count() == 0
+
+    def test_extract_id_not_sequential_previous(self, mock_insights_logger):
+        """ Test when an extract is not sequential to the previous extract, a warning is logged """
+        today_dirname = datetime.now().strftime("%Y-%m-%d")
+
+        filename = f"{today_dirname}/{VALID_DATA_FILE}"
+
+        raw_data = '"NBSSAPPT_HDR"|"00000013"|"20250128"|"170922"|"000001"'
+
+        previous_raw_data = '"NBSSAPPT_HDR"|"00000012"|"20250128"|"170922"|"000001"'
+
+        Command().create_extract(filename, raw_data)
+        
+        with pytest.raises(ExtractValidationError) as error:
+            Command().validate_extract(filename, previous_raw_data)
+            assert str(error.value) == "Extract ID 12 is not sequential to last extract ID 13."
+        
+        assert Extract.objects.count() == 1
+        
+                
+    def test_extract_same_bso_and_extract_id(self, mock_insights_logger):
+        """ Test when an extract has the same extract id and bso code, a warning is logged """
+        today_dirname = datetime.now().strftime("%Y-%m-%d")
+        
+        filename = f"{today_dirname}/{VALID_DATA_FILE}"
+        raw_data = '"NBSSAPPT_HDR"|"00000013"|"20250128"|"170922"|"000001"'
+        Command().create_extract(filename, raw_data)
+
+        same_bso_and_extract_id_raw_data = '"NBSSAPPT_HDR"|"00000013"|"20250128"|"170922"|"000001"'
+        
+        with pytest.raises(ExtractValidationError) as error:
+            Command().validate_extract(filename, same_bso_and_extract_id_raw_data)
+            assert str(error.value) == "Extract ID 13 is not sequential to last extract ID 13."
+        
+    def test_extract_not_sequential_skipped_extract(self, mock_insights_logger):
+        """ Test when an extract is not the next extract in order (i.e. skipped an extract), a warning is logged """
+        today_dirname = datetime.now().strftime("%Y-%m-%d")
+        filename = f"{today_dirname}/{VALID_DATA_FILE}"
+        
+        raw_data = '"NBSSAPPT_HDR"|"00000013"|"20250128"|"170922"|"000001"'
+        
+        Command().create_extract(filename, raw_data)
+        
+        skip_extract_raw_data = '"NBSSAPPT_HDR"|"00000025"|"20250128"|"170922"|"000001"'
+        
+        with pytest.raises(ExtractValidationError) as error:
+            Command().validate_extract(filename, skip_extract_raw_data)
+            assert str(error.value) == "Extract ID 25 is not sequential to last extract ID 13."
+        
+        assert Extract.objects.count() == 1
+