[PRMP-1555] Update Statistical Report to include Audit of access (#594)

Author: NogaNHS

lambdas/handlers/get_report_by_ods_handler.py

@@ -58,13 +58,14 @@ def handle_api_gateway_request(event):
         file_type = FileType.CSV
 
     service = OdsReportService()
-    logger.info(f"Starting process for ods code: {ods_code}")
+    logger.info(f"Received a request to create a report for ODS code: {ods_code}")
     pre_signed_url = service.get_nhs_numbers_by_ods(
         ods_code=ods_code,
         is_pre_signed_needed=True,
         is_upload_to_s3_needed=True,
         file_type_output=file_type,
     )
+    logger.info("A report has been successfully created.")
     return ApiGatewayResponse(
         200, json.dumps({"url": pre_signed_url}), "GET"
     ).create_api_gateway_response()

lambdas/models/report/statistics.py

@@ -58,6 +58,8 @@ class OrganisationData(StatisticData):
     daily_count_deleted: int = 0
     daily_count_searched: int = 0
     daily_count_users_accessing_deceased: int = 0
+    daily_count_ods_report_requested: int = 0
+    daily_count_ods_report_created: int = 0
 
 
 class ApplicationData(StatisticData):

lambdas/services/data_collection_service.py

@@ -24,6 +24,8 @@
     LloydGeorgeRecordsSearched,
     LloydGeorgeRecordsStored,
     LloydGeorgeRecordsViewed,
+    OdsReportsCreated,
+    OdsReportsRequested,
     UniqueActiveUserIds,
 )
 from utils.common_query_filters import UploadCompleted
@@ -198,6 +200,12 @@ def get_organisation_data(
         daily_count_deceased = self.get_cloud_watch_query_result(
             CountUsersAccessedDeceasedPatient, start_date, end_date
         )
+        daily_count_ods_report_requested = self.get_cloud_watch_query_result(
+            OdsReportsRequested, start_date, end_date
+        )
+        daily_count_ods_report_created = self.get_cloud_watch_query_result(
+            OdsReportsCreated, start_date, end_date
+        )
         joined_query_result = self.join_results_by_ods_code(
             [
                 number_of_patients,
@@ -208,6 +216,8 @@ def get_organisation_data(
                 daily_count_stored,
                 daily_count_searched,
                 daily_count_deceased,
+                daily_count_ods_report_requested,
+                daily_count_ods_report_created,
             ]
         )
 

lambdas/services/statistical_report_service.py

@@ -213,6 +213,8 @@ def reorder_columns(self, joined_data: pl.DataFrame) -> pl.DataFrame:
             "weekly_count_downloaded",
             "weekly_count_stored",
             "weekly_count_deleted",
+            "weekly_count_ods_report_requested",
+            "weekly_count_ods_report_created",
             "active_users_count",
             "unique_active_user_ids_hashed",
         ]

lambdas/tests/unit/helpers/data/statistic/mock_collected_data.py

@@ -57,6 +57,8 @@
         daily_count_deleted=2,
         daily_count_searched=30,
         daily_count_users_accessing_deceased=13,
+        daily_count_ods_report_created=13,
+        daily_count_ods_report_requested=20,
     ),
     OrganisationData(
         statistic_id=TEST_UUID,
@@ -70,6 +72,8 @@
         daily_count_deleted=1,
         daily_count_searched=50,
         daily_count_users_accessing_deceased=2,
+        daily_count_ods_report_created=0,
+        daily_count_ods_report_requested=10,
     ),
 ]
 

lambdas/tests/unit/helpers/data/statistic/mock_logs_query_results.py

@@ -84,6 +84,15 @@
     {"ods_code": "H81109", "daily_count_users_accessing_deceased": "13"},
 ]
 
+MOCK_ODS_REPORT_REQUESTED = [
+    {"ods_code": "Y12345", "daily_count_ods_report_requested": "10"},
+    {"ods_code": "H81109", "daily_count_ods_report_requested": "20"},
+]
+
+MOCK_ODS_REPORT_CREATED = [
+    {"ods_code": "Y12345", "daily_count_ods_report_created": "0"},
+    {"ods_code": "H81109", "daily_count_ods_report_created": "13"},
+]
 MOCK_RESPONSE_QUERY_IN_PROGRESS = {"status": "Running"}
 
 MOCK_RESPONSE_QUERY_FAILED = {"status": "Failed"}

lambdas/tests/unit/helpers/data/statistic/mock_statistic_data.py

@@ -93,6 +93,8 @@
     daily_count_downloaded=4,
     daily_count_deleted=1,
     daily_count_users_accessing_deceased=12,
+    daily_count_ods_report_requested=10,
+    daily_count_ods_report_created=0,
 )
 MOCK_ORGANISATION_DATA_2 = OrganisationData(
     statistic_id="9ee2c3d1-97b9-4c34-b75c-83e7d1b442f4",
@@ -105,6 +107,8 @@
     daily_count_downloaded=1,
     daily_count_deleted=1,
     daily_count_users_accessing_deceased=22,
+    daily_count_ods_report_requested=2,
+    daily_count_ods_report_created=2,
 )
 MOCK_ORGANISATION_DATA_3 = OrganisationData(
     statistic_id="3f54cfe3-6c84-4bb2-b5b4-b786aa03b9c7",
@@ -117,6 +121,8 @@
     daily_count_downloaded=5,
     daily_count_deleted=1,
     daily_count_users_accessing_deceased=9,
+    daily_count_ods_report_requested=2,
+    daily_count_ods_report_created=2,
 )
 
 EXPECTED_SUMMARY_ORGANISATION_DATA = pl.DataFrame(
@@ -129,6 +135,8 @@
             "weekly_count_deleted": 1,
             "weekly_count_searched": 0,
             "weekly_count_users_accessing_deceased": 12,
+            "weekly_count_ods_report_requested": 10,
+            "weekly_count_ods_report_created": 0,
             "average_records_per_patient": 4.5,
             "number_of_patients": 4,
         },
@@ -140,6 +148,8 @@
             "weekly_count_deleted": 1 + 1,
             "weekly_count_searched": 0,
             "weekly_count_users_accessing_deceased": 31,
+            "weekly_count_ods_report_requested": 4,
+            "weekly_count_ods_report_created": 4,
             "average_records_per_patient": (3.51 + 2.78) / 2,
             "number_of_patients": 10,
         },
@@ -158,6 +168,7 @@
         "DailyCountDownloaded": 4,
         "DailyCountDeleted": 1,
         "DailyCountUsersAccessingDeceased": 12,
+        "DailyCountOdsReportRequested": 10,
         "StatisticID": "OrganisationData#5acda4bf-8b93-4ba0-8410-789aac4fcbae",
     },
     {
@@ -170,6 +181,8 @@
         "DailyCountDownloaded": 1,
         "DailyCountDeleted": 1,
         "DailyCountUsersAccessingDeceased": 22,
+        "DailyCountOdsReportRequested": 2,
+        "DailyCountOdsReportCreated": 2,
         "StatisticID": "OrganisationData#9ee2c3d1-97b9-4c34-b75c-83e7d1b442f4",
     },
     {
@@ -182,6 +195,8 @@
         "DailyCountDownloaded": 5,
         "DailyCountDeleted": 1,
         "DailyCountUsersAccessingDeceased": 9,
+        "DailyCountOdsReportRequested": 2,
+        "DailyCountOdsReportCreated": 2,
         "StatisticID": "OrganisationData#3f54cfe3-6c84-4bb2-b5b4-b786aa03b9c7",
     },
 ]
@@ -301,6 +316,8 @@
             "Weekly count downloaded": 4,
             "Weekly count stored": 0,
             "Weekly count deleted": 1,
+            "Weekly count ods report requested": 10,
+            "Weekly count ods report created": 0,
             "Active users count": 1,
             "Unique active user ids hashed": str(
                 [str(SERIALISED_APPLICATION_DATA[0]["ActiveUserIdsHashed"][0])]
@@ -321,6 +338,8 @@
             "Weekly count downloaded": 1 + 5,
             "Weekly count stored": 0 + 2,
             "Weekly count deleted": 1 + 1,
+            "Weekly count ods report requested": 4,
+            "Weekly count ods report created": 4,
             "Active users count": 3,
             "Unique active user ids hashed": str(
                 [

lambdas/tests/unit/services/test_data_collection_service.py

@@ -45,6 +45,8 @@
     MOCK_LG_DOWNLOADED,
     MOCK_LG_STORED,
     MOCK_LG_VIEWED,
+    MOCK_ODS_REPORT_CREATED,
+    MOCK_ODS_REPORT_REQUESTED,
     MOCK_PATIENT_SEARCHED,
     MOCK_UNIQUE_ACTIVE_USER_IDS,
 )
@@ -56,6 +58,8 @@
     LloydGeorgeRecordsSearched,
     LloydGeorgeRecordsStored,
     LloydGeorgeRecordsViewed,
+    OdsReportsCreated,
+    OdsReportsRequested,
     UniqueActiveUserIds,
 )
 from utils.common_query_filters import UploadCompleted
@@ -112,6 +116,10 @@ def mock_implementation(query_params: CloudwatchLogsQueryParams, **_kwargs):
             return MOCK_PATIENT_SEARCHED
         elif query_params == CountUsersAccessedDeceasedPatient:
             return MOCK_DECEASED_ACCESS
+        elif query_params == OdsReportsRequested:
+            return MOCK_ODS_REPORT_REQUESTED
+        elif query_params == OdsReportsCreated:
+            return MOCK_ODS_REPORT_CREATED
 
     patched_instance = mocker.patch(
         "services.data_collection_service.CloudwatchService",

lambdas/utils/cloudwatch_logs_query.py

@@ -62,6 +62,24 @@ class CloudwatchLogsQueryParams:
     """,
 )
 
+OdsReportsRequested = CloudwatchLogsQueryParams(
+    lambda_name="GetReportByODS",
+    query_string="""
+        fields @timestamp, Message, Authorisation.selected_organisation.org_ods_code AS ods_code
+        | filter Message like /Received a request to create a report for ODS code/
+        | stats count() AS daily_count_ods_report_requested BY ods_code
+    """,
+)
+
+OdsReportsCreated = CloudwatchLogsQueryParams(
+    lambda_name="GetReportByODS",
+    query_string="""
+        fields @timestamp, Message, Authorisation.selected_organisation.org_ods_code AS ods_code
+        | filter Message = 'A report has been successfully created.'
+        | stats count() AS daily_count_ods_report_created BY ods_code
+    """,
+)
+
 UniqueActiveUserIds = CloudwatchLogsQueryParams(
     lambda_name="TokenRequestHandler",
     query_string="""