Merge pull request #782 from NHSDigital/DTOSS-11710-sequential-extracts

[DTOSS-11710] Add sequential check for Extract

Author: cameronhargreaves1-nhs

manage_breast_screening/notifications/management/commands/create_appointments.py

@@ -23,6 +23,9 @@
 logger = getLogger(__name__)
 
 
+class ExtractValidationError(Exception):
+    pass
+
 class Command(BaseCommand):
     """
     Django Admin command which reads NBSS appointment data from Azure blob storage
@@ -56,9 +59,11 @@ def handle(self, *args, **options):
                 ).readall()
 
                 data_frame = self.raw_data_to_data_frame(blob_content)
+                
+                self.validate_extract(blob.name, blob_content)
 
                 extract = self.create_extract(blob.name, blob_content)
-
+                
                 for idx, row in data_frame.iterrows():
                     if self.is_not_holding_clinic(row):
                         clinic, clinic_created = self.find_or_create_clinic(row)
@@ -72,13 +77,33 @@ def handle(self, *args, **options):
                         extract.appointments.add(appt) if appt is not None else None
 
                 logger.info("Processed %s rows from %s", len(data_frame), blob.name)
+                
+    def validate_extract(self, filename: str, raw_data: str) -> None:
+        bso_code = filename.split("/")[1].split("_")[0]
+        type_id, extract_id, start_date, start_time, record_count = raw_data.split(
+            "\n"
+        )[0].split("|")
+        formatted_extract_id = int(extract_id.replace('"', "").replace("\r", ""))
+        
+        latest_extract = Extract.objects.filter(bso_code = bso_code).order_by("sequence_number").last()
+        
+        if latest_extract:            
+            if formatted_extract_id != (latest_extract.sequence_number + 1):
+                
+                log_msg = "Extract ID %s is not sequential to last extract ID %s." % (
+                    formatted_extract_id,
+                    latest_extract.sequence_number,
+                )
+                
+                raise ExtractValidationError(log_msg)
 
     def create_extract(self, filename: str, raw_data: str) -> Extract:
         bso_code = filename.split("/")[1].split("_")[0]
         type_id, extract_id, start_date, start_time, record_count = raw_data.split(
             "\n"
         )[0].split("|")
         formatted_extract_id = int(extract_id.replace('"', "").replace("\r", ""))
+        
         formatted_record_count = int(record_count.replace('"', "").replace("\r", ""))
 
         return Extract.objects.create(

manage_breast_screening/notifications/tests/management/commands/test_create_appointments.py

@@ -59,7 +59,25 @@ def stored_blob_data(prefix_dir: str, filenames: list[str]):
             mock_blob_contents
         )
         yield
+        
+@contextmanager
+def mocked_blob_storage_contents(prefix_dir: str, filename: str, contents: str):
+    with mocked_blob_storage() as mock_blob_storage:
+        mock_container_client = (
+            mock_blob_storage.return_value.find_or_create_container.return_value
+        )
+        mock_blobs = []
+        mock_blob_contents = []
+        mock_blob = Mock(spec=BlobProperties)
+        mock_blob.name = f"{prefix_dir}/{filename}"
+        mock_blobs.append(mock_blob)
+        mock_blob_contents = contents
 
+        mock_container_client.list_blobs.return_value = mock_blobs
+        mock_container_client.get_blob_client().download_blob().readall.side_effect = (
+            mock_blob_contents
+        )
+        yield
 
 @pytest.mark.django_db
 class TestCreateAppointments:
@@ -391,3 +409,55 @@ def test_errors_with_wrong_format_data(self):
                 Command().handle(**{"date_str": today_dirname})
 
         assert Extract.objects.count() == 0
+
+    def test_extract_id_not_sequential_previous(self, mock_insights_logger):
+        """ Test when an extract is not sequential to the previous extract, a warning is logged """
+        today_dirname = datetime.now().strftime("%Y-%m-%d")
+
+        first_filename = f"{today_dirname}/{VALID_DATA_FILE}"
+
+        raw_data = '"NBSSAPPT_HDR"|"00000013"|"20250128"|"170922"|"000001"'
+
+        previous_raw_data = '"NBSSAPPT_HDR"|"00000012"|"20250128"|"170922"|"000001"'
+
+        # add previous extract so we can test the non sequential extract
+        Command().create_extract(first_filename, raw_data)
+
+        with mocked_blob_storage_contents(today_dirname, "ANOTHER_FILE_NAME.dat", previous_raw_data):
+            with pytest.raises(CommandError) as error:
+                Command().handle(**{"date_str": today_dirname})
+                assert str(error.value) == "Extract ID 12 is not sequential to last extract ID 13."
+
+        assert Extract.objects.count() == 1
+        
+                
+    def test_extract_same_bso_and_extract_id(self, mock_insights_logger):
+        """ Test when an extract has the same extract id and bso code, a warning is logged """
+        today_dirname = datetime.now().strftime("%Y-%m-%d")
+        
+        filename = f"{today_dirname}/{VALID_DATA_FILE}"
+        raw_data = '"NBSSAPPT_HDR"|"00000013"|"20250128"|"170922"|"000001"'
+        Command().create_extract(filename, raw_data)
+
+        same_bso_and_extract_id_raw_data = '"NBSSAPPT_HDR"|"00000013"|"20250128"|"170922"|"000001"'
+        
+        with mocked_blob_storage_contents(today_dirname, "ANOTHER_FILE_NAME.dat", same_bso_and_extract_id_raw_data):
+            with pytest.raises(CommandError) as error:
+                Command().handle(**{"date_str": today_dirname})
+                assert str(error.value) == "Extract ID 13 is not sequential to last extract ID 13."
+        
+    def test_extract_not_sequential_skipped_extract(self, mock_insights_logger):
+        """ Test when an extract is not the next extract in order (i.e. skipped an extract), a warning is logged """
+        today_dirname = datetime.now().strftime("%Y-%m-%d")
+        filename = f"{today_dirname}/{VALID_DATA_FILE}"
+        
+        raw_data = '"NBSSAPPT_HDR"|"00000013"|"20250128"|"170922"|"000001"'
+        
+        Command().create_extract(filename, raw_data)
+        
+        skip_extract_raw_data = '"NBSSAPPT_HDR"|"00000025"|"20250128"|"170922"|"000001"'
+        
+        with mocked_blob_storage_contents(today_dirname, "ANOTHER_FILE_NAME.dat", skip_extract_raw_data):
+            with pytest.raises(CommandError) as error:
+                Command().handle(**{"date_str": today_dirname})
+                assert str(error.value) == "Extract ID 25 is not sequential to last extract ID 13."