diff --git a/.github/workflows/base-lambdas-reusable-deploy-all.yml b/.github/workflows/base-lambdas-reusable-deploy-all.yml index e5bd7798d..5128113a4 100644 --- a/.github/workflows/base-lambdas-reusable-deploy-all.yml +++ b/.github/workflows/base-lambdas-reusable-deploy-all.yml @@ -810,3 +810,17 @@ jobs: lambda_layer_names: "core_lambda_layer,reports_lambda_layer" secrets: AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }} + + deploy_report_distribution_lambda: + name: Deploy Report Distribution + uses: ./.github/workflows/base-lambdas-reusable-deploy.yml + with: + environment: ${{ inputs.environment }} + python_version: ${{ inputs.python_version }} + build_branch: ${{ inputs.build_branch }} + sandbox: ${{ inputs.sandbox }} + lambda_handler_name: report_distribution_handler + lambda_aws_name: reportDistribution + lambda_layer_names: "core_lambda_layer,reports_lambda_layer" + secrets: + AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }} diff --git a/lambdas/handlers/report_distribution_handler.py b/lambdas/handlers/report_distribution_handler.py new file mode 100644 index 000000000..30c8547af --- /dev/null +++ b/lambdas/handlers/report_distribution_handler.py @@ -0,0 +1,61 @@ +import os +from typing import Any, Dict + +from repositories.reporting.report_contact_repository import ReportContactRepository +from services.base.s3_service import S3Service +from services.email_service import EmailService +from services.reporting.report_distribution_service import ReportDistributionService +from utils.audit_logging_setup import LoggingService +from utils.decorators.ensure_env_var import ensure_environment_variables +from utils.decorators.handle_lambda_exceptions import handle_lambda_exceptions +from utils.decorators.override_error_check import override_error_check +from utils.decorators.set_audit_arg import set_request_context_for_logging + +logger = LoggingService(__name__) + + +@ensure_environment_variables( + names=[ + "REPORT_BUCKET_NAME", + "CONTACT_TABLE_NAME", + "PRM_MAILBOX_EMAIL", + "SES_FROM_ADDRESS", + ] +) +@override_error_check +@handle_lambda_exceptions +@set_request_context_for_logging +def lambda_handler(event, context) -> Dict[str, Any]: + action = event.get("action") + if action not in {"list", "process_one"}: + raise ValueError("Invalid action. Expected 'list' or 'process_one'.") + + bucket = event.get("bucket") or os.environ["REPORT_BUCKET_NAME"] + contact_table = os.environ["CONTACT_TABLE_NAME"] + prm_mailbox = os.environ["PRM_MAILBOX_EMAIL"] + from_address = os.environ["SES_FROM_ADDRESS"] + + s3_service = S3Service() + contact_repo = ReportContactRepository(contact_table) + email_service = EmailService() + + service = ReportDistributionService( + s3_service=s3_service, + contact_repo=contact_repo, + email_service=email_service, + bucket=bucket, + from_address=from_address, + prm_mailbox=prm_mailbox, + ) + + if action == "list": + prefix = event["prefix"] + keys = service.list_xlsx_keys(prefix=prefix) + logger.info(f"List mode: returning {len(keys)} key(s) for prefix={prefix}") + return {"bucket": bucket, "prefix": prefix, "keys": keys} + + key = event["key"] + ods_code = service.extract_ods_code_from_key(key) + service.process_one_report(ods_code=ods_code, key=key) + logger.info(f"Process-one mode: processed ods={ods_code}, key={key}") + return {"status": "ok", "bucket": bucket, "key": key, "ods_code": ods_code} diff --git a/lambdas/handlers/report_orchestration_handler.py b/lambdas/handlers/report_orchestration_handler.py index 84f6372ab..be9ea7777 100644 --- a/lambdas/handlers/report_orchestration_handler.py +++ b/lambdas/handlers/report_orchestration_handler.py @@ -1,8 +1,9 @@ import os -import tempfile from datetime import datetime, timedelta, timezone +from typing import Any, Dict from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository +from services.base.s3_service import S3Service from services.reporting.excel_report_generator_service import ExcelReportGenerator from services.reporting.report_orchestration_service import ReportOrchestrationService from utils.audit_logging_setup import LoggingService @@ -23,23 +24,35 @@ def calculate_reporting_window(): yesterday_7am = today_7am - timedelta(days=1) - return ( - int(yesterday_7am.timestamp()), - int(today_7am.timestamp()), - ) + return int(yesterday_7am.timestamp()), int(today_7am.timestamp()) + + +def get_report_date_folder() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%d") + + +def build_s3_key(ods_code: str, report_date: str) -> str: + return f"Report-Orchestration/{report_date}/{ods_code}.xlsx" + @ensure_environment_variables( - names=["BULK_UPLOAD_REPORT_TABLE_NAME"] + names=[ + "BULK_UPLOAD_REPORT_TABLE_NAME", + "REPORT_BUCKET_NAME", + ] ) @override_error_check @handle_lambda_exceptions @set_request_context_for_logging -def lambda_handler(event, context): +def lambda_handler(event, context) -> Dict[str, Any]: logger.info("Report orchestration lambda invoked") - table_name = os.getenv("BULK_UPLOAD_REPORT_TABLE_NAME") + + table_name = os.environ["BULK_UPLOAD_REPORT_TABLE_NAME"] + report_bucket = os.environ["REPORT_BUCKET_NAME"] repository = ReportingDynamoRepository(table_name) excel_generator = ExcelReportGenerator() + s3_service = S3Service() service = ReportOrchestrationService( repository=repository, @@ -47,10 +60,39 @@ def lambda_handler(event, context): ) window_start, window_end = calculate_reporting_window() - tmp_dir = tempfile.mkdtemp() + report_date = get_report_date_folder() + prefix = f"Report-Orchestration/{report_date}/" - service.process_reporting_window( + generated_files = service.process_reporting_window( window_start_ts=window_start, window_end_ts=window_end, - output_dir=tmp_dir, ) + + if not generated_files: + logger.info("No reports generated; exiting") + return { + "report_date": report_date, + "bucket": report_bucket, + "prefix": prefix, + "keys": [], + } + + keys = [] + for ods_code, local_path in generated_files.items(): + key = build_s3_key(ods_code, report_date) + s3_service.upload_file_with_extra_args( + file_name=local_path, + s3_bucket_name=report_bucket, + file_key=key, + extra_args={"ServerSideEncryption": "aws:kms"}, + ) + keys.append(key) + logger.info(f"Uploaded report for ODS={ods_code} to s3://{report_bucket}/{key}") + + logger.info(f"Generated and uploaded {len(keys)} report(s) for report_date={report_date}") + return { + "report_date": report_date, + "bucket": report_bucket, + "prefix": prefix, + "keys": keys, + } diff --git a/lambdas/repositories/reporting/report_contact_repository.py b/lambdas/repositories/reporting/report_contact_repository.py new file mode 100644 index 000000000..84bd06cab --- /dev/null +++ b/lambdas/repositories/reporting/report_contact_repository.py @@ -0,0 +1,17 @@ +from services.base.dynamo_service import DynamoDBService + + +class ReportContactRepository: + def __init__(self, table_name: str): + self.table_name = table_name + self.dynamo = DynamoDBService() + + def get_contact_email(self, ods_code: str) -> str | None: + resp = self.dynamo.get_item( + table_name=self.table_name, + key={"OdsCode": ods_code}, + ) + item = (resp or {}).get("Item") + if not item: + return None + return item.get("Email") diff --git a/lambdas/repositories/reporting/reporting_dynamo_repository.py b/lambdas/repositories/reporting/reporting_dynamo_repository.py index 7a694edf6..86d595d87 100644 --- a/lambdas/repositories/reporting/reporting_dynamo_repository.py +++ b/lambdas/repositories/reporting/reporting_dynamo_repository.py @@ -1,12 +1,13 @@ +from datetime import timedelta from typing import Dict, List -from boto3.dynamodb.conditions import Attr +from boto3.dynamodb.conditions import Key from services.base.dynamo_service import DynamoDBService from utils.audit_logging_setup import LoggingService +from utils.utilities import utc_date_string, utc_date, utc_day_start_timestamp, utc_day_end_timestamp logger = LoggingService(__name__) - class ReportingDynamoRepository: def __init__(self, table_name: str): self.table_name = table_name @@ -17,19 +18,37 @@ def get_records_for_time_window( start_timestamp: int, end_timestamp: int, ) -> List[Dict]: + timestamp_index_name = "TimestampIndex" logger.info( - f"Querying reporting table for window, " - f"table_name: {self.table_name}, " - f"start_timestamp: {start_timestamp}, " - f"end_timestamp: {end_timestamp}", + "Querying reporting table via TimestampIndex for window, " + f"table_name={self.table_name}, start_timestamp={start_timestamp}, end_timestamp={end_timestamp}", ) - filter_expression = Attr("Timestamp").between( - start_timestamp, - end_timestamp, - ) + start_date = utc_date(start_timestamp) + end_date = utc_date(end_timestamp) - return self.dynamo_service.scan_whole_table( - table_name=self.table_name, - filter_expression=filter_expression, - ) + records_for_window: List[Dict] = [] + current_date = start_date + + while current_date <= end_date: + day_start_ts = utc_day_start_timestamp(current_date) + day_end_ts = utc_day_end_timestamp(current_date) + + effective_start_ts = max(start_timestamp, day_start_ts) + effective_end_ts = min(end_timestamp, day_end_ts) + + key_condition = ( + Key("Date").eq(current_date.isoformat()) + & Key("Timestamp").between(effective_start_ts, effective_end_ts) + ) + + records_for_day = self.dynamo_service.query_by_key_condition_expression( + table_name=self.table_name, + index_name=timestamp_index_name, + key_condition_expression=key_condition, + ) + + records_for_window.extend(records_for_day) + current_date += timedelta(days=1) + + return records_for_window \ No newline at end of file diff --git a/lambdas/requirements/layers/requirements_reports_lambda_layer.txt b/lambdas/requirements/layers/requirements_reports_lambda_layer.txt index 44709d749..872e8ad16 100644 --- a/lambdas/requirements/layers/requirements_reports_lambda_layer.txt +++ b/lambdas/requirements/layers/requirements_reports_lambda_layer.txt @@ -1,2 +1,3 @@ openpyxl==3.1.5 -reportlab==4.3.1 \ No newline at end of file +reportlab==4.3.1 +pyzipper==0.3.6 \ No newline at end of file diff --git a/lambdas/services/base/dynamo_service.py b/lambdas/services/base/dynamo_service.py index 7f8584220..ff278b1f7 100644 --- a/lambdas/services/base/dynamo_service.py +++ b/lambdas/services/base/dynamo_service.py @@ -433,6 +433,46 @@ def build_update_transaction_item( } } + def query_by_key_condition_expression( + self, + table_name: str, + key_condition_expression: ConditionBase, + index_name: str | None = None, + query_filter: Attr | ConditionBase | None = None, + limit: int | None = None, + ) -> list[dict]: + table = self.get_table(table_name) + + collected_items: list[dict] = [] + exclusive_start_key: dict | None = None + + while True: + query_params: dict = {"KeyConditionExpression": key_condition_expression} + + if index_name: + query_params["IndexName"] = index_name + if query_filter: + query_params["FilterExpression"] = query_filter + if exclusive_start_key: + query_params["ExclusiveStartKey"] = exclusive_start_key + if limit: + query_params["Limit"] = limit + + try: + response = table.query(**query_params) + except ClientError as exc: + logger.error(str(exc), {"Result": f"Unable to query table: {table_name}"}) + raise + + collected_items.extend(response.get("Items", [])) + + exclusive_start_key = response.get("LastEvaluatedKey") + if not exclusive_start_key: + break + + return collected_items + + def query_table_with_paginator( self, table_name: str, diff --git a/lambdas/services/email_service.py b/lambdas/services/email_service.py new file mode 100644 index 000000000..8a11a9bd8 --- /dev/null +++ b/lambdas/services/email_service.py @@ -0,0 +1,113 @@ +import boto3 +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from email.mime.application import MIMEApplication +from typing import Iterable, Optional + +from utils.audit_logging_setup import LoggingService + +logger = LoggingService(__name__) + + +class EmailService: + """ + General email sender via SES (AWS Simple Email Service) Raw Email (supports attachments). + Higher-level methods prepare inputs and call send_email(). + """ + + def __init__(self): + self.ses = boto3.client("ses") + + def send_email( + self, + *, + to_address: str, + subject: str, + body_text: str, + from_address: str, + attachments: Optional[Iterable[str]] = None, + )->MIMEMultipart: + msg = MIMEMultipart() + msg["Subject"] = subject + msg["To"] = to_address + msg["From"] = from_address + + msg.attach(MIMEText(body_text, "plain")) + + for attachment_path in attachments or []: + with open(attachment_path, "rb") as f: + part = MIMEApplication(f.read()) + part.add_header( + "Content-Disposition", + "attachment", + filename=attachment_path.split("/")[-1], + ) + msg.attach(part) + logger.info( + f"Sending email: from={from_address!r}, to={to_address!r}, subject={subject!r}, " + f"attachments={len(list(attachments or []))}" + ) + return self._send_raw(msg, to_address) + + def _send_raw(self, msg: MIMEMultipart, to_address: str)->MIMEMultipart: + subject = msg.get("Subject", "") + from_address = msg.get("From", "") + logger.info(f"Sending SES raw email: subject={subject!r}, from={from_address!r}, to={to_address!r}") + resp = self.ses.send_raw_email( + Source=from_address, + RawMessage={"Data": msg.as_string()}, + Destinations=[to_address], + ) + + logger.info(f"SES accepted email: subject={subject!r}, message_id={resp.get('MessageId')}") + return resp + + def send_report_email( + self, + *, + to_address: str, + from_address: str, + attachment_path: str, + ): + self.send_email( + to_address=to_address, + from_address=from_address, + subject="Daily Upload Report", + body_text="Please find your encrypted daily upload report attached.", + attachments=[attachment_path], + ) + + def send_password_email( + self, + *, + to_address: str, + from_address: str, + password: str, + ): + self.send_email( + to_address=to_address, + from_address=from_address, + subject="Daily Upload Report Password", + body_text=f"Password for your report:\n\n{password}", + ) + + def send_prm_missing_contact_email( + self, + *, + prm_mailbox: str, + from_address: str, + ods_code: str, + attachment_path: str, + password: str, + ): + self.send_email( + to_address=prm_mailbox, + from_address=from_address, + subject=f"Missing contact for ODS {ods_code}", + body_text=( + f"No contact found for ODS {ods_code}.\n\n" + f"Password: {password}\n\n" + f"Please resolve the contact and forward the report." + ), + attachments=[attachment_path], + ) diff --git a/lambdas/services/reporting/report_distribution_service.py b/lambdas/services/reporting/report_distribution_service.py new file mode 100644 index 000000000..f64c632c6 --- /dev/null +++ b/lambdas/services/reporting/report_distribution_service.py @@ -0,0 +1,121 @@ +import os +import secrets +import tempfile +from typing import List + +import boto3 + +from repositories.reporting.report_contact_repository import ReportContactRepository +from services.base.s3_service import S3Service +from services.email_service import EmailService +from utils.audit_logging_setup import LoggingService +from utils.zip_utils import zip_encrypt_file + +logger = LoggingService(__name__) + + +class ReportDistributionService: + def __init__( + self, + *, + s3_service: S3Service, + contact_repo: ReportContactRepository, + email_service: EmailService, + bucket: str, + from_address: str, + prm_mailbox: str, + ): + self.s3_service = s3_service + self.contact_repo = contact_repo + self.email_service = email_service + self.bucket = bucket + self.from_address = from_address + self.prm_mailbox = prm_mailbox + self._s3_client = boto3.client("s3") + + @staticmethod + def extract_ods_code_from_key(key: str) -> str: + filename = key.split("/")[-1] + return filename[:-5] if filename.lower().endswith(".xlsx") else filename + + def list_xlsx_keys(self, prefix: str) -> List[str]: + paginator = self._s3_client.get_paginator("list_objects_v2") + keys: List[str] = [] + + for page in paginator.paginate(Bucket=self.bucket, Prefix=prefix): + for obj in page.get("Contents", []): + key = obj["Key"] + if key.endswith(".xlsx"): + keys.append(key) + + return keys + + def process_one_report(self, *, ods_code: str, key: str) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + local_xlsx = os.path.join(tmpdir, f"{ods_code}.xlsx") + local_zip = os.path.join(tmpdir, f"{ods_code}.zip") + + self.s3_service.download_file(self.bucket, key, local_xlsx) + + password = secrets.token_urlsafe(16) + zip_encrypt_file( + input_path=local_xlsx, + output_zip=local_zip, + password=password, + ) + + self.send_report_emails( + ods_code=ods_code, + attachment_path=local_zip, + password=password, + ) + + def send_report_emails(self, *, ods_code: str, attachment_path: str, password: str) -> None: + try: + contact_email = self.contact_repo.get_contact_email(ods_code) + except Exception as e: + logger.exception( + f"Contact lookup failed for ODS={ods_code}; falling back to PRM. Error: {e}" + ) + contact_email = None + + if contact_email: + logger.info(f"Contact found for ODS={ods_code}, emailing {contact_email}") + self.email_contact( + to_address=contact_email, + attachment_path=attachment_path, + password=password, + ) + return + + logger.info(f"No contact found for ODS={ods_code}, sending to PRM mailbox") + self.email_prm_missing_contact( + ods_code=ods_code, + attachment_path=attachment_path, + password=password, + ) + + def email_contact(self, *, to_address: str, attachment_path: str, password: str) -> None: + logger.info(f"Sending report email to {to_address}") + self.email_service.send_report_email( + to_address=to_address, + from_address=self.from_address, + attachment_path=attachment_path, + ) + logger.info(f"Sending password email to {to_address}") + self.email_service.send_password_email( + to_address=to_address, + from_address=self.from_address, + password=password, + ) + + def email_prm_missing_contact( + self, *, ods_code: str, attachment_path: str, password: str + ) -> None: + self.email_service.send_prm_missing_contact_email( + prm_mailbox=self.prm_mailbox, + from_address=self.from_address, + ods_code=ods_code, + attachment_path=attachment_path, + password=password, + ) diff --git a/lambdas/services/reporting/report_orchestration_service.py b/lambdas/services/reporting/report_orchestration_service.py index a9500735b..69a919cc6 100644 --- a/lambdas/services/reporting/report_orchestration_service.py +++ b/lambdas/services/reporting/report_orchestration_service.py @@ -1,5 +1,6 @@ import tempfile from collections import defaultdict +from typing import Dict from utils.audit_logging_setup import LoggingService @@ -7,11 +8,7 @@ class ReportOrchestrationService: - def __init__( - self, - repository, - excel_generator, - ): + def __init__(self, repository, excel_generator): self.repository = repository self.excel_generator = excel_generator @@ -19,24 +16,28 @@ def process_reporting_window( self, window_start_ts: int, window_end_ts: int, - output_dir: str, - ): + ) -> Dict[str, str]: records = self.repository.get_records_for_time_window( window_start_ts, window_end_ts, ) + if not records: logger.info("No records found for reporting window") - return + return {} records_by_ods = self.group_records_by_ods(records) + generated_files: Dict[str, str] = {} for ods_code, ods_records in records_by_ods.items(): logger.info( - f"Generating report for ODS ods_code = {ods_code} record_count = {len(ods_records)}" + f"Generating report for ODS={ods_code}, records={len(ods_records)}" ) - self.generate_ods_report(ods_code, ods_records) - logger.info("Report orchestration completed") + file_path = self.generate_ods_report(ods_code, ods_records) + generated_files[ods_code] = file_path + + logger.info(f"Generated {len(generated_files)} report(s)") + return generated_files @staticmethod def group_records_by_ods(records: list[dict]) -> dict[str, list[dict]]: @@ -46,11 +47,7 @@ def group_records_by_ods(records: list[dict]) -> dict[str, list[dict]]: grouped[ods_code].append(record) return grouped - def generate_ods_report( - self, - ods_code: str, - records: list[dict], - ): + def generate_ods_report(self, ods_code: str, records: list[dict]) -> str: with tempfile.NamedTemporaryFile( suffix=f"_{ods_code}.xlsx", delete=False, @@ -60,3 +57,4 @@ def generate_ods_report( records=records, output_path=tmp.name, ) + return tmp.name diff --git a/lambdas/tests/unit/handlers/test_report_distribution_handler.py b/lambdas/tests/unit/handlers/test_report_distribution_handler.py new file mode 100644 index 000000000..d129d2ed3 --- /dev/null +++ b/lambdas/tests/unit/handlers/test_report_distribution_handler.py @@ -0,0 +1,130 @@ +import importlib +import os +import pytest + +MODULE_UNDER_TEST = "handlers.report_distribution_handler" + + +@pytest.fixture +def handler_module(): + return importlib.import_module(MODULE_UNDER_TEST) + + +@pytest.fixture +def required_env(mocker): + mocker.patch.dict( + os.environ, + { + "REPORT_BUCKET_NAME": "my-report-bucket", + "CONTACT_TABLE_NAME": "contact-table", + "PRM_MAILBOX_EMAIL": "prm@example.com", + "SES_FROM_ADDRESS": "from@example.com", + }, + clear=False, + ) + + +def test_lambda_handler_wires_dependencies_and_returns_result_list_mode( + mocker, handler_module, required_env +): + event = {"action": "list", "prefix": "reports/2026-01-01/"} + context = mocker.Mock() + context.aws_request_id = "req-123" # avoid JSON serialization issues in decorators + + s3_instance = mocker.Mock(name="S3ServiceInstance") + contact_repo_instance = mocker.Mock(name="ReportContactRepositoryInstance") + email_instance = mocker.Mock(name="EmailServiceInstance") + + svc_instance = mocker.Mock(name="ReportDistributionServiceInstance") + svc_instance.list_xlsx_keys.return_value = ["a.xlsx", "b.xlsx"] + + mocked_s3_cls = mocker.patch.object( + handler_module, "S3Service", autospec=True, return_value=s3_instance + ) + mocked_contact_repo_cls = mocker.patch.object( + handler_module, + "ReportContactRepository", + autospec=True, + return_value=contact_repo_instance, + ) + mocked_email_cls = mocker.patch.object( + handler_module, "EmailService", autospec=True, return_value=email_instance + ) + mocked_dist_svc_cls = mocker.patch.object( + handler_module, + "ReportDistributionService", + autospec=True, + return_value=svc_instance, + ) + + result = handler_module.lambda_handler(event, context) + + mocked_s3_cls.assert_called_once_with() + mocked_contact_repo_cls.assert_called_once_with("contact-table") + mocked_email_cls.assert_called_once_with() + + mocked_dist_svc_cls.assert_called_once_with( + s3_service=s3_instance, + contact_repo=contact_repo_instance, + email_service=email_instance, + bucket="my-report-bucket", + from_address="from@example.com", + prm_mailbox="prm@example.com", + ) + + svc_instance.list_xlsx_keys.assert_called_once_with(prefix="reports/2026-01-01/") + assert result == { + "bucket": "my-report-bucket", + "prefix": "reports/2026-01-01/", + "keys": ["a.xlsx", "b.xlsx"], + } + + +def test_lambda_handler_uses_bucket_from_event_when_provided_list_mode( + mocker, handler_module, required_env +): + event = {"action": "list", "prefix": "p/", "bucket": "override-bucket"} + context = mocker.Mock() + context.aws_request_id = "req-456" + + svc_instance = mocker.Mock() + svc_instance.list_xlsx_keys.return_value = [] + + mocker.patch.object(handler_module, "ReportDistributionService", autospec=True, return_value=svc_instance) + mocker.patch.object(handler_module, "S3Service", autospec=True, return_value=mocker.Mock()) + mocker.patch.object(handler_module, "ReportContactRepository", autospec=True, return_value=mocker.Mock()) + mocker.patch.object(handler_module, "EmailService", autospec=True, return_value=mocker.Mock()) + + result = handler_module.lambda_handler(event, context) + + svc_instance.list_xlsx_keys.assert_called_once_with(prefix="p/") + assert result == {"bucket": "override-bucket", "prefix": "p/", "keys": []} + + +def test_lambda_handler_process_one_mode_happy_path( + mocker, handler_module, required_env +): + event = {"action": "process_one", "key": "reports/ABC/whatever.xlsx"} + context = mocker.Mock() + context.aws_request_id = "req-789" + + svc_instance = mocker.Mock() + svc_instance.extract_ods_code_from_key.return_value = "ABC" + svc_instance.process_one_report.return_value = None + + mocker.patch.object(handler_module, "ReportDistributionService", autospec=True, return_value=svc_instance) + mocker.patch.object(handler_module, "S3Service", autospec=True, return_value=mocker.Mock()) + mocker.patch.object(handler_module, "ReportContactRepository", autospec=True, return_value=mocker.Mock()) + mocker.patch.object(handler_module, "EmailService", autospec=True, return_value=mocker.Mock()) + + result = handler_module.lambda_handler(event, context) + + svc_instance.extract_ods_code_from_key.assert_called_once_with("reports/ABC/whatever.xlsx") + svc_instance.process_one_report.assert_called_once_with(ods_code="ABC", key="reports/ABC/whatever.xlsx") + + assert result == { + "status": "ok", + "bucket": "my-report-bucket", + "key": "reports/ABC/whatever.xlsx", + "ods_code": "ABC", + } diff --git a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py index c9ee2ff84..10364c5e4 100644 --- a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py +++ b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py @@ -1,6 +1,9 @@ -from unittest import mock +import json +import os from unittest.mock import MagicMock + import pytest + from handlers.report_orchestration_handler import lambda_handler @@ -9,8 +12,15 @@ class FakeContext: @pytest.fixture(autouse=True) -def mock_env(monkeypatch): - monkeypatch.setenv("BULK_UPLOAD_REPORT_TABLE_NAME", "TestTable") +def mock_env(mocker): + mocker.patch.dict( + os.environ, + { + "BULK_UPLOAD_REPORT_TABLE_NAME": "TestTable", + "REPORT_BUCKET_NAME": "test-report-bucket", + }, + clear=False, + ) @pytest.fixture @@ -19,7 +29,7 @@ def mock_logger(mocker): @pytest.fixture -def mock_repo(mocker): +def mock_repo_cls(mocker): return mocker.patch( "handlers.report_orchestration_handler.ReportingDynamoRepository", autospec=True, @@ -27,7 +37,7 @@ def mock_repo(mocker): @pytest.fixture -def mock_excel_generator(mocker): +def mock_excel_generator_cls(mocker): return mocker.patch( "handlers.report_orchestration_handler.ExcelReportGenerator", autospec=True, @@ -35,7 +45,15 @@ def mock_excel_generator(mocker): @pytest.fixture -def mock_service(mocker): +def mock_s3_service_cls(mocker): + return mocker.patch( + "handlers.report_orchestration_handler.S3Service", + autospec=True, + ) + + +@pytest.fixture +def mock_service_cls(mocker): return mocker.patch( "handlers.report_orchestration_handler.ReportOrchestrationService", autospec=True, @@ -50,25 +68,142 @@ def mock_window(mocker): ) -def test_lambda_handler_calls_service( - mock_logger, mock_repo, mock_excel_generator, mock_service, mock_window +@pytest.fixture +def mock_report_date(mocker): + return mocker.patch( + "handlers.report_orchestration_handler.get_report_date_folder", + return_value="2026-01-02", + ) + + +def test_lambda_handler_wires_dependencies_and_calls_service( + mock_logger, + mock_repo_cls, + mock_excel_generator_cls, + mock_s3_service_cls, + mock_service_cls, + mock_window, + mock_report_date, ): - lambda_handler(event={}, context=FakeContext()) + service_instance = mock_service_cls.return_value + service_instance.process_reporting_window.return_value = { + "A12345": "/tmp/A12345.xlsx", + "B67890": "/tmp/B67890.xlsx", + } - mock_repo.assert_called_once_with("TestTable") - mock_excel_generator.assert_called_once_with() + result = lambda_handler(event={}, context=FakeContext()) + + mock_repo_cls.assert_called_once_with("TestTable") + mock_excel_generator_cls.assert_called_once_with() + mock_s3_service_cls.assert_called_once_with() + + mock_service_cls.assert_called_once_with( + repository=mock_repo_cls.return_value, + excel_generator=mock_excel_generator_cls.return_value, + ) - mock_service.assert_called_once() - instance = mock_service.return_value - instance.process_reporting_window.assert_called_once_with( + service_instance.process_reporting_window.assert_called_once_with( window_start_ts=100, window_end_ts=200, - output_dir=mock.ANY, ) mock_logger.info.assert_any_call("Report orchestration lambda invoked") + assert result["report_date"] == "2026-01-02" + assert result["bucket"] == "test-report-bucket" + assert result["prefix"] == "Report-Orchestration/2026-01-02/" + assert set(result["keys"]) == { + "Report-Orchestration/2026-01-02/A12345.xlsx", + "Report-Orchestration/2026-01-02/B67890.xlsx", + } + + +def test_lambda_handler_calls_window_function( + mock_service_cls, + mock_window, + mock_report_date, + mock_s3_service_cls, +): + mock_service_cls.return_value.process_reporting_window.return_value = {} -def test_lambda_handler_calls_window_function(mock_service, mock_window): lambda_handler(event={}, context=FakeContext()) + mock_window.assert_called_once() + + +def test_lambda_handler_returns_empty_keys_when_no_reports_generated( + mock_service_cls, + mock_logger, + mock_s3_service_cls, + mock_window, + mock_report_date, +): + mock_service_cls.return_value.process_reporting_window.return_value = {} + + result = lambda_handler(event={}, context=FakeContext()) + + assert result == { + "report_date": "2026-01-02", + "bucket": "test-report-bucket", + "prefix": "Report-Orchestration/2026-01-02/", + "keys": [], + } + + mock_s3_service_cls.return_value.upload_file_with_extra_args.assert_not_called() + mock_logger.info.assert_any_call("No reports generated; exiting") + + +def test_lambda_handler_uploads_each_report_to_s3_with_kms_encryption( + mock_service_cls, + mock_s3_service_cls, + mock_report_date, + mock_window, +): + mock_service_cls.return_value.process_reporting_window.return_value = { + "A12345": "/tmp/A12345.xlsx", + "UNKNOWN": "/tmp/UNKNOWN.xlsx", + } + + result = lambda_handler(event={}, context=FakeContext()) + + s3_instance = mock_s3_service_cls.return_value + assert s3_instance.upload_file_with_extra_args.call_count == 2 + + s3_instance.upload_file_with_extra_args.assert_any_call( + file_name="/tmp/A12345.xlsx", + s3_bucket_name="test-report-bucket", + file_key="Report-Orchestration/2026-01-02/A12345.xlsx", + extra_args={"ServerSideEncryption": "aws:kms"}, + ) + + s3_instance.upload_file_with_extra_args.assert_any_call( + file_name="/tmp/UNKNOWN.xlsx", + s3_bucket_name="test-report-bucket", + file_key="Report-Orchestration/2026-01-02/UNKNOWN.xlsx", + extra_args={"ServerSideEncryption": "aws:kms"}, + ) + + assert result["keys"] == [ + "Report-Orchestration/2026-01-02/A12345.xlsx", + "Report-Orchestration/2026-01-02/UNKNOWN.xlsx", + ] + + +def test_lambda_handler_returns_error_when_required_env_missing(mocker): + mocker.patch.dict(os.environ, {"BULK_UPLOAD_REPORT_TABLE_NAME": "TestTable"}, clear=False) + os.environ.pop("REPORT_BUCKET_NAME", None) + + ctx = FakeContext() + ctx.aws_request_id = "test-request-id" + + result = lambda_handler(event={}, context=ctx) + + assert isinstance(result, dict) + assert result["statusCode"] == 500 + + body = json.loads(result["body"]) + assert body["err_code"] == "ENV_5001" + assert "REPORT_BUCKET_NAME" in body["message"] + + if body.get("interaction_id") is not None: + assert body["interaction_id"] == "test-request-id" diff --git a/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py b/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py new file mode 100644 index 000000000..d53426732 --- /dev/null +++ b/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py @@ -0,0 +1,63 @@ +import pytest + +from repositories.reporting.report_contact_repository import ReportContactRepository + + +@pytest.fixture +def mock_dynamo(mocker): + mock = mocker.Mock() + mocker.patch( + "repositories.reporting.report_contact_repository.DynamoDBService", + return_value=mock, + ) + return mock + + +@pytest.fixture +def repo(mock_dynamo): + return ReportContactRepository(table_name="report-contacts") + + +def test_get_contact_email_returns_email_when_item_exists(repo, mock_dynamo): + mock_dynamo.get_item.return_value = { + "Item": { + "OdsCode": "Y12345", + "Email": "contact@example.com", + } + } + + result = repo.get_contact_email("Y12345") + + mock_dynamo.get_item.assert_called_once_with( + table_name="report-contacts", + key={"OdsCode": "Y12345"}, + ) + assert result == "contact@example.com" + + +def test_get_contact_email_returns_none_when_item_missing(repo, mock_dynamo): + mock_dynamo.get_item.return_value = {} # or None + + result = repo.get_contact_email("Y12345") + + mock_dynamo.get_item.assert_called_once_with( + table_name="report-contacts", + key={"OdsCode": "Y12345"}, + ) + assert result is None + + +def test_get_contact_email_returns_none_when_email_missing(repo, mock_dynamo): + mock_dynamo.get_item.return_value = { + "Item": { + "OdsCode": "Y12345", + } + } + + result = repo.get_contact_email("Y12345") + + mock_dynamo.get_item.assert_called_once_with( + table_name="report-contacts", + key={"OdsCode": "Y12345"}, + ) + assert result is None diff --git a/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py b/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py index df3c55304..5697eb112 100644 --- a/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py +++ b/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py @@ -1,3 +1,4 @@ +from datetime import date from unittest.mock import MagicMock import pytest @@ -6,12 +7,12 @@ @pytest.fixture def mock_dynamo_service(mocker): - mock_service = mocker.patch( + mock_service_class = mocker.patch( "repositories.reporting.reporting_dynamo_repository.DynamoDBService" ) - instance = mock_service.return_value - instance.scan_whole_table = MagicMock() - return instance + mock_instance = mock_service_class.return_value + mock_instance.query_by_key_condition_expression = MagicMock() + return mock_instance @pytest.fixture @@ -19,23 +20,45 @@ def reporting_repo(mock_dynamo_service): return ReportingDynamoRepository(table_name="TestTable") -def test_get_records_for_time_window_calls_scan(mock_dynamo_service, reporting_repo): - mock_dynamo_service.scan_whole_table.return_value = [] +def test_get_records_for_time_window_same_date_queries_once(mocker, mock_dynamo_service, reporting_repo): + mock_utc_date = mocker.patch("repositories.reporting.reporting_dynamo_repository.utc_date") + mock_utc_date.side_effect = [date(2026, 1, 7), date(2026, 1, 7)] - reporting_repo.get_records_for_time_window(100, 200) + mock_dynamo_service.query_by_key_condition_expression.return_value = [{"ID": "one"}] - mock_dynamo_service.scan_whole_table.assert_called_once() - assert "filter_expression" in mock_dynamo_service.scan_whole_table.call_args.kwargs + result = reporting_repo.get_records_for_time_window(100, 200) + assert result == [{"ID": "one"}] + mock_dynamo_service.query_by_key_condition_expression.assert_called_once() -def test_get_records_for_time_window_returns_empty_list( - mock_dynamo_service, reporting_repo -): - start_ts = 0 - end_ts = 50 - mock_dynamo_service.scan_whole_table.return_value = [] + call_kwargs = mock_dynamo_service.query_by_key_condition_expression.call_args.kwargs + assert call_kwargs["table_name"] == "TestTable" + assert call_kwargs["index_name"] == "TimestampIndex" + assert "key_condition_expression" in call_kwargs - result = reporting_repo.get_records_for_time_window(start_ts, end_ts) + +def test_get_records_for_time_window_different_dates_queries_twice(mocker, mock_dynamo_service, reporting_repo): + mock_utc_date = mocker.patch("repositories.reporting.reporting_dynamo_repository.utc_date") + mock_utc_date.side_effect = [date(2026, 1, 6), date(2026, 1, 7)] + + mock_dynamo_service.query_by_key_condition_expression.side_effect = [ + [{"ID": "start-day"}], + [{"ID": "end-day"}], + ] + + result = reporting_repo.get_records_for_time_window(100, 200) + + assert result == [{"ID": "start-day"}, {"ID": "end-day"}] + assert mock_dynamo_service.query_by_key_condition_expression.call_count == 2 + + +def test_get_records_for_time_window_returns_empty_list_when_no_items(mocker, mock_dynamo_service, reporting_repo): + mock_utc_date = mocker.patch("repositories.reporting.reporting_dynamo_repository.utc_date") + mock_utc_date.side_effect = [date(2026, 1, 6), date(2026, 1, 7)] + + mock_dynamo_service.query_by_key_condition_expression.side_effect = [[], []] + + result = reporting_repo.get_records_for_time_window(100, 200) assert result == [] - mock_dynamo_service.scan_whole_table.assert_called_once() + assert mock_dynamo_service.query_by_key_condition_expression.call_count == 2 diff --git a/lambdas/tests/unit/services/base/test_dynamo_service.py b/lambdas/tests/unit/services/base/test_dynamo_service.py index 6d66c6673..3c6afbcf6 100755 --- a/lambdas/tests/unit/services/base/test_dynamo_service.py +++ b/lambdas/tests/unit/services/base/test_dynamo_service.py @@ -1207,6 +1207,118 @@ def test_build_key_condition_non_matching_list_lengths( search_key=search_key, search_condition=search_condition ) +def test_query_by_key_condition_expression_single_page_returns_items( + mock_service, mock_table +): + key_condition_expression = Key("Date").eq("2026-01-07") & Key("Timestamp").gte(1767779952) + + mock_table.return_value.query.return_value = { + "Items": [{"ID": "item-1"}, {"ID": "item-2"}] + } + + result = mock_service.query_by_key_condition_expression( + table_name=MOCK_TABLE_NAME, + index_name="TimestampIndex", + key_condition_expression=key_condition_expression, + ) + + assert result == [{"ID": "item-1"}, {"ID": "item-2"}] + mock_table.assert_called_with(MOCK_TABLE_NAME) + mock_table.return_value.query.assert_called_once_with( + IndexName="TimestampIndex", + KeyConditionExpression=key_condition_expression, + ) + + +def test_query_by_key_condition_expression_handles_pagination( + mock_service, mock_table +): + key_condition_expression = Key("Date").eq("2026-01-07") & Key("Timestamp").gte(1767779952) + + mock_table.return_value.query.side_effect = [ + { + "Items": [{"ID": "item-1"}], + "LastEvaluatedKey": {"ID": "page-2"}, + }, + { + "Items": [{"ID": "item-2"}], + "LastEvaluatedKey": {"ID": "page-3"}, + }, + { + "Items": [{"ID": "item-3"}], + }, + ] + + result = mock_service.query_by_key_condition_expression( + table_name=MOCK_TABLE_NAME, + index_name="TimestampIndex", + key_condition_expression=key_condition_expression, + ) + + assert result == [{"ID": "item-1"}, {"ID": "item-2"}, {"ID": "item-3"}] + mock_table.assert_called_with(MOCK_TABLE_NAME) + + expected_calls = [ + call( + IndexName="TimestampIndex", + KeyConditionExpression=key_condition_expression, + ), + call( + IndexName="TimestampIndex", + KeyConditionExpression=key_condition_expression, + ExclusiveStartKey={"ID": "page-2"}, + ), + call( + IndexName="TimestampIndex", + KeyConditionExpression=key_condition_expression, + ExclusiveStartKey={"ID": "page-3"}, + ), + ] + mock_table.return_value.query.assert_has_calls(expected_calls) + + +def test_query_by_key_condition_expression_passes_filter_and_limit( + mock_service, mock_table +): + key_condition_expression = Key("Date").eq("2026-01-07") & Key("Timestamp").lte(1767780025) + filter_expression = Attr("UploadStatus").eq("complete") + + mock_table.return_value.query.return_value = {"Items": [{"ID": "item-1"}]} + + result = mock_service.query_by_key_condition_expression( + table_name=MOCK_TABLE_NAME, + index_name="TimestampIndex", + key_condition_expression=key_condition_expression, + query_filter=filter_expression, + limit=25, + ) + + assert result == [{"ID": "item-1"}] + mock_table.assert_called_with(MOCK_TABLE_NAME) + mock_table.return_value.query.assert_called_once_with( + IndexName="TimestampIndex", + KeyConditionExpression=key_condition_expression, + FilterExpression=filter_expression, + Limit=25, + ) + + +def test_query_by_key_condition_expression_client_error_raises_exception( + mock_service, mock_table +): + key_condition_expression = Key("Date").eq("2026-01-07") & Key("Timestamp").gte(1767779952) + + mock_table.return_value.query.side_effect = MOCK_CLIENT_ERROR + + with pytest.raises(ClientError) as exc_info: + mock_service.query_by_key_condition_expression( + table_name=MOCK_TABLE_NAME, + index_name="TimestampIndex", + key_condition_expression=key_condition_expression, + ) + + assert exc_info.value == MOCK_CLIENT_ERROR + def test_query_table_using_paginator(mock_service): mock_paginator = mock_service.client.get_paginator.return_value = MagicMock() diff --git a/lambdas/tests/unit/services/reporting/test_email_service.py b/lambdas/tests/unit/services/reporting/test_email_service.py new file mode 100644 index 000000000..efde7f725 --- /dev/null +++ b/lambdas/tests/unit/services/reporting/test_email_service.py @@ -0,0 +1,154 @@ +import pytest + +from services.email_service import EmailService + + +@pytest.fixture +def email_service(mocker): + mocker.patch("services.email_service.boto3.client", autospec=True) + svc = EmailService() + svc.ses = mocker.Mock() + return svc + +def test_send_email_sends_raw_email_without_attachments(email_service, mocker): + mocked_send_raw = mocker.patch.object(email_service, "_send_raw", autospec=True) + + email_service.send_email( + to_address="to@example.com", + subject="Hello", + body_text="Body text", + from_address="from@example.com", + attachments=None, + ) + + mocked_send_raw.assert_called_once() + + call_args, call_kwargs = mocked_send_raw.call_args + assert call_kwargs == {} + + msg_arg = call_args[0] + to_arg = call_args[1] + + assert to_arg == "to@example.com" + assert msg_arg["Subject"] == "Hello" + assert msg_arg["To"] == "to@example.com" + assert msg_arg["From"] == "from@example.com" + + raw = msg_arg.as_string() + assert "Body text" in raw + + + +def test_send_email_attaches_files_and_sets_filenames(email_service, mocker): + file_bytes_1 = b"zipbytes1" + file_bytes_2 = b"zipbytes2" + + m1 = mocker.mock_open(read_data=file_bytes_1) + m2 = mocker.mock_open(read_data=file_bytes_2) + + mocked_open = mocker.patch("services.email_service.open", create=True) + mocked_open.side_effect = [m1.return_value, m2.return_value] + + mocked_send_raw = mocker.patch.object(email_service, "_send_raw", autospec=True) + + email_service.send_email( + to_address="to@example.com", + subject="With Attachments", + body_text="See attached", + from_address="from@example.com", + attachments=["/tmp/a.zip", "/var/tmp/b.zip"], + ) + + assert mocked_open.call_count == 2 + mocked_open.assert_any_call("/tmp/a.zip", "rb") + mocked_open.assert_any_call("/var/tmp/b.zip", "rb") + + mocked_send_raw.assert_called_once() + + call_args, call_kwargs = mocked_send_raw.call_args + assert call_kwargs == {} + msg = call_args[0] + raw = msg.as_string() + + assert 'filename="a.zip"' in raw + assert 'filename="b.zip"' in raw + assert "See attached" in raw + + + +def test_send_raw_calls_ses_send_raw_email(email_service, mocker): + from email.mime.multipart import MIMEMultipart + msg = MIMEMultipart() + msg["Subject"] = "S" + msg["To"] = "to@example.com" + msg["From"] = "from@example.com" + + email_service._send_raw(msg, "to@example.com") + + email_service.ses.send_raw_email.assert_called_once() + call_kwargs = email_service.ses.send_raw_email.call_args.kwargs + + assert call_kwargs["Destinations"] == ["to@example.com"] + assert "RawMessage" in call_kwargs + assert "Data" in call_kwargs["RawMessage"] + assert isinstance(call_kwargs["RawMessage"]["Data"], str) + assert "Subject: S" in call_kwargs["RawMessage"]["Data"] + + +def test_send_report_email_calls_send_email_with_expected_inputs(email_service, mocker): + mocked_send_email = mocker.patch.object(email_service, "send_email", autospec=True) + + email_service.send_report_email( + to_address="to@example.com", + from_address="from@example.com", + attachment_path="/tmp/report.zip", + ) + + mocked_send_email.assert_called_once_with( + to_address="to@example.com", + from_address="from@example.com", + subject="Daily Upload Report", + body_text="Please find your encrypted daily upload report attached.", + attachments=["/tmp/report.zip"], + ) + + +def test_send_password_email_calls_send_email_with_expected_inputs(email_service, mocker): + mocked_send_email = mocker.patch.object(email_service, "send_email", autospec=True) + + email_service.send_password_email( + to_address="to@example.com", + from_address="from@example.com", + password="pw123", + ) + + mocked_send_email.assert_called_once_with( + to_address="to@example.com", + from_address="from@example.com", + subject="Daily Upload Report Password", + body_text="Password for your report:\n\npw123", + ) + + +def test_send_prm_missing_contact_email_calls_send_email_with_expected_inputs(email_service, mocker): + mocked_send_email = mocker.patch.object(email_service, "send_email", autospec=True) + + email_service.send_prm_missing_contact_email( + prm_mailbox="prm@example.com", + from_address="from@example.com", + ods_code="Y12345", + attachment_path="/tmp/report.zip", + password="pw123", + ) + + mocked_send_email.assert_called_once_with( + to_address="prm@example.com", + from_address="from@example.com", + subject="Missing contact for ODS Y12345", + body_text=( + "No contact found for ODS Y12345.\n\n" + "Password: pw123\n\n" + "Please resolve the contact and forward the report." + ), + attachments=["/tmp/report.zip"], + ) diff --git a/lambdas/tests/unit/services/reporting/test_report_distribution_service.py b/lambdas/tests/unit/services/reporting/test_report_distribution_service.py new file mode 100644 index 000000000..56b5150e0 --- /dev/null +++ b/lambdas/tests/unit/services/reporting/test_report_distribution_service.py @@ -0,0 +1,341 @@ +import os +import pytest + +from services.reporting.report_distribution_service import ReportDistributionService + +@pytest.fixture +def mock_s3_service(mocker): + return mocker.Mock() + + +@pytest.fixture +def mock_contact_repo(mocker): + repo = mocker.Mock() + repo.get_contact_email.return_value = None + return repo + + +@pytest.fixture +def mock_email_service(mocker): + return mocker.Mock() + + +@pytest.fixture +def service(mocker, mock_s3_service, mock_contact_repo, mock_email_service): + mocker.patch("services.reporting.report_distribution_service.boto3.client", autospec=True) + + return ReportDistributionService( + s3_service=mock_s3_service, + contact_repo=mock_contact_repo, + email_service=mock_email_service, + bucket="my-bucket", + from_address="from@example.com", + prm_mailbox="prm@example.com", + ) + +def test_extract_ods_code_from_key_strips_xlsx_extension(): + assert ReportDistributionService.extract_ods_code_from_key( + "Report-Orchestration/2026-01-01/Y12345.xlsx" + ) == "Y12345" + + +def test_extract_ods_code_from_key_is_case_insensitive(): + assert ReportDistributionService.extract_ods_code_from_key("a/b/C789.XLSX") == "C789" + + +def test_extract_ods_code_from_key_keeps_non_xlsx_filename(): + assert ReportDistributionService.extract_ods_code_from_key("a/b/report.csv") == "report.csv" + +def test_list_xlsx_keys_filters_only_xlsx(service, mocker): + paginator = mocker.Mock() + paginator.paginate.return_value = [ + { + "Contents": [ + {"Key": "Report-Orchestration/2026-01-01/A123.xlsx"}, + {"Key": "Report-Orchestration/2026-01-01/readme.txt"}, + {"Key": "Report-Orchestration/2026-01-01/B456.xls"}, + {"Key": "Report-Orchestration/2026-01-01/C789.xlsx"}, + ] + }, + {"Contents": [{"Key": "Report-Orchestration/2026-01-01/D000.xlsx"}]}, + {}, + ] + + service._s3_client.get_paginator.return_value = paginator + + keys = service.list_xlsx_keys(prefix="Report-Orchestration/2026-01-01/") + + assert keys == [ + "Report-Orchestration/2026-01-01/A123.xlsx", + "Report-Orchestration/2026-01-01/C789.xlsx", + "Report-Orchestration/2026-01-01/D000.xlsx", + ] + + service._s3_client.get_paginator.assert_called_once_with("list_objects_v2") + paginator.paginate.assert_called_once_with( + Bucket="my-bucket", + Prefix="Report-Orchestration/2026-01-01/", + ) + + +def test_list_xlsx_keys_returns_empty_when_no_objects(service, mocker): + paginator = mocker.Mock() + paginator.paginate.return_value = [{"Contents": []}, {}] + service._s3_client.get_paginator.return_value = paginator + + keys = service.list_xlsx_keys(prefix="Report-Orchestration/2026-01-01/") + + assert keys == [] + + +def test_list_xlsx_keys_skips_pages_without_contents(service, mocker): + paginator = mocker.Mock() + paginator.paginate.return_value = [{}, {"Contents": [{"Key": "p/X.xlsx"}]}] + service._s3_client.get_paginator.return_value = paginator + + keys = service.list_xlsx_keys(prefix="p/") + + assert keys == ["p/X.xlsx"] + + +def test_process_one_report_downloads_encrypts_and_delegates_email( + service, mocker, mock_s3_service +): + mocker.patch( + "services.reporting.report_distribution_service.secrets.token_urlsafe", + return_value="fixed-password", + ) + + fake_tmp = "/tmp/fake_tmpdir" + td = mocker.MagicMock() + td.__enter__.return_value = fake_tmp + td.__exit__.return_value = False + mocker.patch( + "services.reporting.report_distribution_service.tempfile.TemporaryDirectory", + return_value=td, + ) + + mocked_zip = mocker.patch( + "services.reporting.report_distribution_service.zip_encrypt_file", + autospec=True, + ) + mocked_send = mocker.patch.object(service, "send_report_emails", autospec=True) + + service.process_one_report( + ods_code="Y12345", + key="Report-Orchestration/2026-01-01/Y12345.xlsx", + ) + + local_xlsx = os.path.join(fake_tmp, "Y12345.xlsx") + local_zip = os.path.join(fake_tmp, "Y12345.zip") + + mock_s3_service.download_file.assert_called_once_with( + "my-bucket", + "Report-Orchestration/2026-01-01/Y12345.xlsx", + local_xlsx, + ) + + mocked_zip.assert_called_once_with( + input_path=local_xlsx, + output_zip=local_zip, + password="fixed-password", + ) + + mocked_send.assert_called_once_with( + ods_code="Y12345", + attachment_path=local_zip, + password="fixed-password", + ) + + +def test_process_one_report_propagates_download_errors(service, mocker, mock_s3_service): + mock_s3_service.download_file.side_effect = RuntimeError("download failed") + + td = mocker.MagicMock() + td.__enter__.return_value = "/tmp/fake_tmpdir" + td.__exit__.return_value = False + mocker.patch( + "services.reporting.report_distribution_service.tempfile.TemporaryDirectory", + return_value=td, + ) + + mocked_zip = mocker.patch("services.reporting.report_distribution_service.zip_encrypt_file", autospec=True) + mocked_send = mocker.patch.object(service, "send_report_emails", autospec=True) + + with pytest.raises(RuntimeError, match="download failed"): + service.process_one_report(ods_code="Y12345", key="k.xlsx") + + mocked_zip.assert_not_called() + mocked_send.assert_not_called() + + +def test_process_one_report_does_not_send_email_if_zip_fails(service, mocker, mock_s3_service): + mocker.patch( + "services.reporting.report_distribution_service.secrets.token_urlsafe", + return_value="pw", + ) + + td = mocker.MagicMock() + td.__enter__.return_value = "/tmp/fake_tmpdir" + td.__exit__.return_value = False + mocker.patch( + "services.reporting.report_distribution_service.tempfile.TemporaryDirectory", + return_value=td, + ) + + mocker.patch( + "services.reporting.report_distribution_service.zip_encrypt_file", + side_effect=RuntimeError("zip failed"), + autospec=True, + ) + mocked_send = mocker.patch.object(service, "send_report_emails", autospec=True) + + with pytest.raises(RuntimeError, match="zip failed"): + service.process_one_report(ods_code="Y12345", key="k.xlsx") + + mocked_send.assert_not_called() + + +def test_process_one_report_does_not_zip_or_send_email_if_password_generation_fails( + service, mocker, mock_s3_service +): + mocker.patch( + "services.reporting.report_distribution_service.secrets.token_urlsafe", + side_effect=RuntimeError("secrets failed"), + ) + + fake_tmp = "/tmp/fake_tmpdir" + td = mocker.MagicMock() + td.__enter__.return_value = fake_tmp + td.__exit__.return_value = False + mocker.patch( + "services.reporting.report_distribution_service.tempfile.TemporaryDirectory", + return_value=td, + ) + + mocked_zip = mocker.patch( + "services.reporting.report_distribution_service.zip_encrypt_file", + autospec=True, + ) + mocked_send = mocker.patch.object(service, "send_report_emails", autospec=True) + + with pytest.raises(RuntimeError, match="secrets failed"): + service.process_one_report(ods_code="Y12345", key="k.xlsx") + + mock_s3_service.download_file.assert_called_once_with( + "my-bucket", + "k.xlsx", + os.path.join(fake_tmp, "Y12345.xlsx"), + ) + + mocked_zip.assert_not_called() + mocked_send.assert_not_called() + +def test_send_report_emails_with_contact_calls_email_contact(service, mock_contact_repo, mocker): + mock_contact_repo.get_contact_email.return_value = "contact@example.com" + + mocked_email_contact = mocker.patch.object(service, "email_contact", autospec=True) + mocked_email_prm = mocker.patch.object(service, "email_prm_missing_contact", autospec=True) + + service.send_report_emails( + ods_code="Y12345", + attachment_path="/tmp/Y12345.zip", + password="pw", + ) + + mock_contact_repo.get_contact_email.assert_called_once_with("Y12345") + mocked_email_contact.assert_called_once_with( + to_address="contact@example.com", + attachment_path="/tmp/Y12345.zip", + password="pw", + ) + mocked_email_prm.assert_not_called() + + +def test_send_report_emails_without_contact_calls_email_prm(service, mock_contact_repo, mocker): + mock_contact_repo.get_contact_email.return_value = None + + mocked_email_contact = mocker.patch.object(service, "email_contact", autospec=True) + mocked_email_prm = mocker.patch.object(service, "email_prm_missing_contact", autospec=True) + + service.send_report_emails( + ods_code="A99999", + attachment_path="/tmp/A99999.zip", + password="pw", + ) + + mock_contact_repo.get_contact_email.assert_called_once_with("A99999") + mocked_email_prm.assert_called_once_with( + ods_code="A99999", + attachment_path="/tmp/A99999.zip", + password="pw", + ) + mocked_email_contact.assert_not_called() + + +def test_send_report_emails_contact_lookup_exception_falls_back_to_prm(service, mock_contact_repo, mocker): + mock_contact_repo.get_contact_email.side_effect = RuntimeError("ddb down") + + mocked_email_contact = mocker.patch.object(service, "email_contact", autospec=True) + mocked_email_prm = mocker.patch.object(service, "email_prm_missing_contact", autospec=True) + + service.send_report_emails( + ods_code="A99999", + attachment_path="/tmp/A99999.zip", + password="pw", + ) + + mocked_email_contact.assert_not_called() + mocked_email_prm.assert_called_once_with( + ods_code="A99999", + attachment_path="/tmp/A99999.zip", + password="pw", + ) + + +def test_email_contact_sends_report_and_password(service, mock_email_service): + service.email_contact( + to_address="contact@example.com", + attachment_path="/tmp/file.zip", + password="pw", + ) + + mock_email_service.send_report_email.assert_called_once_with( + to_address="contact@example.com", + from_address="from@example.com", + attachment_path="/tmp/file.zip", + ) + mock_email_service.send_password_email.assert_called_once_with( + to_address="contact@example.com", + from_address="from@example.com", + password="pw", + ) + + +def test_email_contact_sends_password_even_if_report_email_fails(service, mock_email_service): + mock_email_service.send_report_email.side_effect = RuntimeError("SES down") + + with pytest.raises(RuntimeError, match="SES down"): + service.email_contact( + to_address="contact@example.com", + attachment_path="/tmp/file.zip", + password="pw", + ) + + mock_email_service.send_password_email.assert_not_called() + + +def test_email_prm_missing_contact_sends_prm_missing_contact_email(service, mock_email_service): + service.email_prm_missing_contact( + ods_code="X11111", + attachment_path="/tmp/file.zip", + password="pw", + ) + + mock_email_service.send_prm_missing_contact_email.assert_called_once_with( + prm_mailbox="prm@example.com", + from_address="from@example.com", + ods_code="X11111", + attachment_path="/tmp/file.zip", + password="pw", + ) diff --git a/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py index c068ba0c1..baf54577f 100644 --- a/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py +++ b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py @@ -1,4 +1,5 @@ import pytest + from services.reporting.report_orchestration_service import ReportOrchestrationService @@ -22,36 +23,27 @@ def report_orchestration_service(mock_repository, mock_excel_generator): ) -def test_process_reporting_window_no_records( +def test_process_reporting_window_no_records_returns_empty_dict_and_does_not_generate( report_orchestration_service, mock_repository, mock_excel_generator ): mock_repository.get_records_for_time_window.return_value = [] - report_orchestration_service.process_reporting_window(100, 200, output_dir="/tmp") + result = report_orchestration_service.process_reporting_window(100, 200) + assert result == {} mock_excel_generator.create_report_orchestration_xlsx.assert_not_called() -def test_group_records_by_ods_groups_correctly(): - records = [ - {"UploaderOdsCode": "Y12345", "ID": 1}, - {"UploaderOdsCode": "Y12345", "ID": 2}, - {"UploaderOdsCode": "A99999", "ID": 3}, - {"ID": 4}, # missing ODS - {"UploaderOdsCode": None, "ID": 5}, # null ODS - ] +def test_process_reporting_window_calls_repository_with_window_args( + report_orchestration_service, mock_repository, mocker +): + mock_repository.get_records_for_time_window.return_value = [{"UploaderOdsCode": "X1", "ID": 1}] + mocked_generate = mocker.patch.object(report_orchestration_service, "generate_ods_report", return_value="/tmp/x.xlsx") - result = ReportOrchestrationService.group_records_by_ods(records) + report_orchestration_service.process_reporting_window(100, 200) - assert result["Y12345"] == [ - {"UploaderOdsCode": "Y12345", "ID": 1}, - {"UploaderOdsCode": "Y12345", "ID": 2}, - ] - assert result["A99999"] == [{"UploaderOdsCode": "A99999", "ID": 3}] - assert result["UNKNOWN"] == [ - {"ID": 4}, - {"UploaderOdsCode": None, "ID": 5}, - ] + mock_repository.get_records_for_time_window.assert_called_once_with(100, 200) + mocked_generate.assert_called_once() def test_process_reporting_window_generates_reports_per_ods( @@ -65,10 +57,10 @@ def test_process_reporting_window_generates_reports_per_ods( mock_repository.get_records_for_time_window.return_value = records mocked_generate = mocker.patch.object( - report_orchestration_service, "generate_ods_report" + report_orchestration_service, "generate_ods_report", return_value="/tmp/ignored.xlsx" ) - report_orchestration_service.process_reporting_window(100, 200, output_dir="/tmp") + report_orchestration_service.process_reporting_window(100, 200) mocked_generate.assert_any_call( "Y12345", @@ -84,21 +76,115 @@ def test_process_reporting_window_generates_reports_per_ods( assert mocked_generate.call_count == 2 -def test_generate_ods_report_creates_excel_report( +def test_process_reporting_window_returns_mapping_of_ods_to_generated_file_path( + report_orchestration_service, mock_repository, mocker +): + records = [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"UploaderOdsCode": "A99999", "ID": 2}, + ] + mock_repository.get_records_for_time_window.return_value = records + + def _side_effect(ods_code, ods_records): + return f"/tmp/{ods_code}.xlsx" + + mocker.patch.object( + report_orchestration_service, + "generate_ods_report", + side_effect=_side_effect, + ) + + result = report_orchestration_service.process_reporting_window(100, 200) + + assert result == { + "Y12345": "/tmp/Y12345.xlsx", + "A99999": "/tmp/A99999.xlsx", + } + + +def test_process_reporting_window_includes_unknown_ods_group( + report_orchestration_service, mock_repository, mocker +): + records = [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"ID": 2}, # missing ODS -> UNKNOWN + {"UploaderOdsCode": None, "ID": 3}, # null ODS -> UNKNOWN + ] + mock_repository.get_records_for_time_window.return_value = records + + mocked_generate = mocker.patch.object( + report_orchestration_service, "generate_ods_report", return_value="/tmp/ignored.xlsx" + ) + + report_orchestration_service.process_reporting_window(100, 200) + + # Expect 2 groups: Y12345 and UNKNOWN + assert mocked_generate.call_count == 2 + mocked_generate.assert_any_call( + "Y12345", + [{"UploaderOdsCode": "Y12345", "ID": 1}], + ) + mocked_generate.assert_any_call( + "UNKNOWN", + [{"ID": 2}, {"UploaderOdsCode": None, "ID": 3}], + ) + + +def test_group_records_by_ods_groups_correctly(): + records = [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"UploaderOdsCode": "Y12345", "ID": 2}, + {"UploaderOdsCode": "A99999", "ID": 3}, + {"ID": 4}, # missing ODS + {"UploaderOdsCode": None, "ID": 5}, # null ODS + ] + + result = ReportOrchestrationService.group_records_by_ods(records) + + assert result["Y12345"] == [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"UploaderOdsCode": "Y12345", "ID": 2}, + ] + assert result["A99999"] == [{"UploaderOdsCode": "A99999", "ID": 3}] + assert result["UNKNOWN"] == [ + {"ID": 4}, + {"UploaderOdsCode": None, "ID": 5}, + ] + + +def test_group_records_by_ods_empty_input_returns_empty_mapping(): + result = ReportOrchestrationService.group_records_by_ods([]) + assert dict(result) == {} + + +def test_group_records_by_ods_treats_empty_string_as_unknown(): + records = [{"UploaderOdsCode": "", "ID": 1}] + result = ReportOrchestrationService.group_records_by_ods(records) + assert result["UNKNOWN"] == [{"UploaderOdsCode": "", "ID": 1}] + + +def test_generate_ods_report_creates_excel_report_and_returns_path( report_orchestration_service, mock_excel_generator, mocker ): fake_tmp = mocker.MagicMock() fake_tmp.__enter__.return_value = fake_tmp fake_tmp.name = "/tmp/fake_Y12345.xlsx" - mocker.patch( + mocked_ntf = mocker.patch( "services.reporting.report_orchestration_service.tempfile.NamedTemporaryFile", return_value=fake_tmp, ) records = [{"ID": 1, "UploaderOdsCode": "Y12345"}] - report_orchestration_service.generate_ods_report("Y12345", records) + result_path = report_orchestration_service.generate_ods_report("Y12345", records) + + assert result_path == fake_tmp.name + + mocked_ntf.assert_called_once_with( + suffix="_Y12345.xlsx", + delete=False, + ) mock_excel_generator.create_report_orchestration_xlsx.assert_called_once_with( ods_code="Y12345", diff --git a/lambdas/tests/unit/utils/test_utilities.py b/lambdas/tests/unit/utils/test_utilities.py index 68ddb60e5..23a3b3745 100755 --- a/lambdas/tests/unit/utils/test_utilities.py +++ b/lambdas/tests/unit/utils/test_utilities.py @@ -12,7 +12,7 @@ get_pds_service, parse_date, redact_id_to_last_4_chars, - validate_nhs_number, + validate_nhs_number, utc_date_string, ) @@ -135,3 +135,18 @@ def test_format_cloudfront_url_valid(): def test_parse_date_returns_correct_date_for_valid_formats(input_date, expected_date): result = parse_date(input_date) assert result == expected_date + +@pytest.mark.parametrize( + "timestamp_seconds, expected_date_string", + [ + (0, "1970-01-01"), + (1704067200, "2024-01-01"), + (1767780025, "2026-01-07"), + (1704153599, "2024-01-01"), + (1704153600, "2024-01-02"), + ], +) +def test_utc_date_string_returns_correct_utc_date( + timestamp_seconds, expected_date_string +): + assert utc_date_string(timestamp_seconds) == expected_date_string diff --git a/lambdas/utils/utilities.py b/lambdas/utils/utilities.py index b83af68e4..bb2de83ea 100755 --- a/lambdas/utils/utilities.py +++ b/lambdas/utils/utilities.py @@ -2,7 +2,7 @@ import os import re import uuid -from datetime import datetime +from datetime import datetime, timezone, date, time from urllib.parse import urlparse from inflection import camelize @@ -127,3 +127,17 @@ def parse_date(date_string: str) -> datetime | None: except ValueError: continue return None + + +def utc_date_string(timestamp_seconds: int) -> str: + return datetime.fromtimestamp(timestamp_seconds, tz=timezone.utc).strftime("%Y-%m-%d") + +def utc_date(timestamp_seconds: int) -> date: + return datetime.fromtimestamp(timestamp_seconds, tz=timezone.utc).date() + +def utc_day_start_timestamp(day: date) -> int: + return int( + datetime.combine(day, time.min, tzinfo=timezone.utc).timestamp() + ) +def utc_day_end_timestamp(day: date) -> int: + return utc_day_start_timestamp(day) + 24 * 60 * 60 - 1 \ No newline at end of file diff --git a/lambdas/utils/zip_utils.py b/lambdas/utils/zip_utils.py new file mode 100644 index 000000000..a09e8934f --- /dev/null +++ b/lambdas/utils/zip_utils.py @@ -0,0 +1,20 @@ +import os +import pyzipper + + +def zip_encrypt_file(*, input_path: str, output_zip: str, password: str) -> None: + """ + Create an AES-encrypted ZIP file containing a single file. + + :param input_path: Path to the file to zip + :param output_zip: Path of the zip file to create + :param password: Password for AES encryption + """ + with pyzipper.AESZipFile( + output_zip, + "w", + compression=pyzipper.ZIP_DEFLATED, + encryption=pyzipper.WZ_AES, + ) as zf: + zf.setpassword(password.encode("utf-8")) + zf.write(input_path, arcname=os.path.basename(input_path))