[PRMP-1064] - Update bulk upload summary and ODS reports to contain t… (#457)

* [PRMP-1064] - Update bulk upload summary and ODS reports to contain total ingested

* [prmp-1064] - refactor code to make total_ingested a set and add to it in the currently existing loops

Author: abid-nhs

lambdas/models/bulk_upload_report_output.py

@@ -19,6 +19,7 @@ def __init__(
         self.total_suspended = set()
         self.total_deceased = set()
         self.total_restricted = set()
+        self.total_ingested = set()
 
     def get_total_successful_nhs_numbers(self) -> list:
         if self.total_successful:
@@ -40,6 +41,9 @@ def get_total_deceased_count(self) -> int:
     def get_total_restricted_count(self) -> int:
         return len(self.total_restricted)
 
+    def get_total_ingested_count(self) -> int:
+        return len(self.total_ingested)
+
     @staticmethod
     def get_sorted(to_sort: set) -> list:
         return sorted(to_sort, key=lambda x: x[0]) if to_sort else []
@@ -64,6 +68,7 @@ def populate_report(self):
         logger.info(f"Generating ODS report file for {self.uploader_ods_code}")
 
         for item in self.report_items:
+            self.total_ingested.add(item.nhs_number)
             if item.upload_status == UploadStatus.COMPLETE:
                 self.process_successful_report_item(item)
             elif item.upload_status == UploadStatus.FAILED:
@@ -143,6 +148,7 @@ def populate_report(self):
         ods_code_success_total = {}
 
         for report in self.ods_reports:
+            self.total_ingested.update(report.total_ingested)
             self.total_successful.update(report.total_successful)
             self.total_registered_elsewhere.update(report.total_registered_elsewhere)
             self.total_suspended.update(report.total_suspended)

lambdas/services/bulk_upload_report_service.py

@@ -86,6 +86,7 @@ def generate_individual_ods_report(
 
         self.write_summary_data_to_csv(
             file_name=file_name,
+            total_ingested=ods_report.get_total_ingested_count(),
             total_successful=ods_report.get_total_successful_count(),
             total_registered_elsewhere=ods_report.get_total_registered_elsewhere_count(),
             total_suspended=ods_report.get_total_suspended_count(),
@@ -113,6 +114,7 @@ def generate_summary_report(self, ods_reports: list[OdsReport]):
 
         self.write_summary_data_to_csv(
             file_name=file_name,
+            total_ingested=summary_report.get_total_ingested_count(),
             total_successful=summary_report.get_total_successful_count(),
             total_registered_elsewhere=summary_report.get_total_registered_elsewhere_count(),
             total_suspended=summary_report.get_total_suspended_count(),
@@ -287,6 +289,7 @@ def write_items_to_csv(items: list[BulkUploadReport], csv_file_path: str):
     @staticmethod
     def write_summary_data_to_csv(
         file_name: str,
+        total_ingested: int,
         total_successful: int,
         total_registered_elsewhere: int,
         total_suspended: int,
@@ -298,6 +301,7 @@ def write_summary_data_to_csv(
             writer = csv.writer(output_file)
 
             writer.writerow(["Type", "Description", "Count"])
+            writer.writerow(["Total", "Total Ingested", total_ingested])
             writer.writerow(["Total", "Total Successful", total_successful])
             writer.writerow(
                 [

lambdas/tests/unit/helpers/data/bulk_upload/expected_bulk_upload_summary_report.csv

@@ -1,4 +1,5 @@
 Type,Description,Count
+Total,Total Ingested,16
 Total,Total Successful,10
 Total,Successful - Registered Elsewhere,2
 Total,Successful - Suspended,2

lambdas/tests/unit/helpers/data/bulk_upload/expected_ods_report_for_uploader_1.csv

@@ -1,4 +1,5 @@
 Type,Description,Count
+Total,Total Ingested,8
 Total,Total Successful,5
 Total,Successful - Registered Elsewhere,1
 Total,Successful - Suspended,1

lambdas/tests/unit/helpers/data/bulk_upload/expected_ods_report_for_uploader_2.csv

@@ -1,4 +1,5 @@
 Type,Description,Count
+Total,Total Ingested,8
 Total,Total Successful,5
 Total,Successful - Registered Elsewhere,1
 Total,Successful - Suspended,1

lambdas/tests/unit/models/test_bulk_upload_report_output.py

@@ -78,6 +78,16 @@ def test_report_base_get_sorted_returns_empty():
 def test_ods_report_populate_report_populates_successfully():
     expected = {
         "generated_at": get_timestamp(),
+        "total_ingested": {
+            "9000000000",
+            "9000000001",
+            "9000000002",
+            "9000000003",
+            "9000000004",
+            "9000000005",
+            "9000000006",
+            "9000000007",
+        },
         "total_successful": {
             ("9000000000", "2012-01-13"),
             ("9000000001", "2012-01-13"),
@@ -211,6 +221,7 @@ def test_ods_report_get_unsuccessful_reasons_data_rows_returns_correct_rows():
 def test_ods_report_populate_report_empty_list_populates_successfully():
     expected = {
         "generated_at": get_timestamp(),
+        "total_ingested": set(),
         "total_successful": set(),
         "total_registered_elsewhere": set(),
         "total_suspended": set(),
@@ -275,6 +286,24 @@ def test_summary_report_populate_report_populates_successfully():
 
     expected = {
         "generated_at": get_timestamp(),
+        "total_ingested": {
+            "9000000006",
+            "9000000009",
+            "9000000005",
+            "9000000010",
+            "9000000013",
+            "9000000016",
+            "9000000004",
+            "9000000007",
+            "9000000012",
+            "9000000011",
+            "9000000002",
+            "9000000003",
+            "9000000001",
+            "9000000000",
+            "9000000014",
+            "9000000015",
+        },
         "total_successful": {
             ("9000000000", "2012-01-13"),
             ("9000000001", "2012-01-13"),
@@ -336,6 +365,7 @@ def test_summary_report_populate_report_empty_reports_objects_populate_successfu
 
     expected = {
         "generated_at": get_timestamp(),
+        "total_ingested": set(),
         "total_successful": set(),
         "total_registered_elsewhere": set(),
         "total_suspended": set(),
@@ -359,6 +389,7 @@ def test_summary_report_populate_report_empty_reports_objects_populate_successfu
 def test_summary_report_populate_report_no_report_objects_populate_successfully():
     expected = {
         "generated_at": get_timestamp(),
+        "total_ingested": set(),
         "total_successful": set(),
         "total_registered_elsewhere": set(),
         "total_suspended": set(),

lambdas/tests/unit/services/test_bulk_upload_report_service.py

@@ -357,6 +357,7 @@ def test_generate_individual_ods_report_creates_ods_report(
 
     mock_write_summary_data_to_csv.assert_called_with(
         file_name=f"daily_statistical_report_bulk_upload_ods_summary_{MOCK_TIMESTAMP}_uploaded_by_{TEST_CURRENT_GP_ODS}.csv",
+        total_ingested=8,
         total_successful=5,
         total_registered_elsewhere=1,
         total_suspended=1,