Feature/eli 138 audit record within lambda (#214)

* Patch - Initial test and AuditService stub.

* Patch - Commit 1 Adds flask g and before and after request audit

* Patch - Added audit record logic.

* Patch - Fix broken tests.

* Adding integration test for audit

* Fixed actions part of the audit

* Added unit tests for AuditContext

* Added unit tests and fixed TODOs

* Fixed TODOs

* Fixed linting

* Uses list[SuggestedAction] instead of SuggestedActions

* Adds internal action code to audit actions

* Fix todo's

* Fix linting again

* Fixing priority type

* Cleaning audit bucket after use per every test run

* Fix lint

* Fix format and lint

* Convert audit_log into camelCase when writing record to firehose

* github role permissions - reduced (#217)

* Adds audit package for separation of concerns

---------

Co-authored-by: Shweta <[email protected]>
Co-authored-by: Karthikeyannhs <[email protected]>

Author: 3 people

src/eligibility_signposting_api/app.py

@@ -7,7 +7,7 @@
 from mangum import Mangum
 from mangum.types import LambdaContext, LambdaEvent
 
-from eligibility_signposting_api import repos, services
+from eligibility_signposting_api import audit, repos, services
 from eligibility_signposting_api.config.config import config, init_logging
 from eligibility_signposting_api.error_handler import handle_exception
 from eligibility_signposting_api.views import eligibility_blueprint
@@ -41,7 +41,9 @@ def create_app() -> Flask:
     app.register_error_handler(Exception, handle_exception)
 
     # Set up dependency injection using wireup
-    container = wireup.create_sync_container(service_modules=[services, repos], parameters={**app.config, **config()})
+    container = wireup.create_sync_container(
+        service_modules=[services, repos, audit], parameters={**app.config, **config()}
+    )
     wireup.integration.flask.setup(container, app)
 
     logger.info("app ready", extra={"config": {**app.config, **config()}})

src/eligibility_signposting_api/audit/__init__.py

@@ -0,0 +1,146 @@
+import logging
+from datetime import UTC, datetime
+from operator import attrgetter
+from uuid import UUID
+
+from flask import Request, g
+
+from eligibility_signposting_api.audit.audit_models import (
+    AuditAction,
+    AuditCondition,
+    AuditEligibilityCohortGroups,
+    AuditEligibilityCohorts,
+    AuditEvent,
+    AuditFilterRule,
+    AuditRedirectRule,
+    AuditSuitabilityRule,
+    RequestAuditData,
+    RequestAuditHeader,
+    RequestAuditQueryParams,
+)
+from eligibility_signposting_api.audit.audit_service import AuditService
+from eligibility_signposting_api.model.eligibility import (
+    CohortGroupResult,
+    ConditionName,
+    IterationResult,
+    Status,
+    SuggestedAction,
+)
+from eligibility_signposting_api.model.rules import CampaignID, CampaignVersion, Iteration, RuleName, RulePriority
+
+logger = logging.getLogger(__name__)
+
+
+class AuditContext:
+    @staticmethod
+    def add_request_details(request: Request) -> None:
+        g.audit_log = AuditEvent()
+        resource_id = None
+        if request.view_args and request.view_args["nhs_number"]:
+            resource_id = request.view_args["nhs_number"]
+        g.audit_log.request = RequestAuditData(
+            nhs_number=resource_id,
+            request_timestamp=datetime.now(tz=UTC),
+            headers=(
+                RequestAuditHeader(
+                    x_request_id=request.headers.get("X-Request-ID"),
+                    x_correlation_id=request.headers.get("X-Correlation-ID"),
+                    nhsd_end_user_organisation_ods=request.headers.get("NHSD-End-User-Organisation-ODS"),
+                    nhsd_application_id=request.headers.get("nhsd-application-id"),
+                )
+            ),
+            query_params=(
+                RequestAuditQueryParams(
+                    category=request.args.get("category"),
+                    conditions=request.args.get("conditions"),
+                    include_actions=request.args.get("includeActions"),
+                )
+            ),
+        )
+
+    @staticmethod
+    def append_audit_condition(
+        suggested_actions: list[SuggestedAction] | None,
+        condition_name: ConditionName,
+        best_results: tuple[Iteration | None, IterationResult | None, dict[str, CohortGroupResult] | None],
+        campaign_details: tuple[CampaignID | None, CampaignVersion | None],
+        redirect_rule_details: tuple[RulePriority | None, RuleName | None],
+    ) -> None:
+        audit_eligibility_cohorts, audit_eligibility_cohort_groups, audit_actions = [], [], []
+        audit_filter_rule, audit_suitability_rule, audit_redirect_rule = None, None, None
+        best_active_iteration = best_results[0]
+        best_candidate = best_results[1]
+        best_cohort_results = best_results[2]
+
+        if best_cohort_results:
+            for value in sorted(best_cohort_results.values(), key=attrgetter("cohort_code")):
+                cohort_status_name = value.status.name if value.status else None
+                audit_eligibility_cohorts.append(
+                    AuditEligibilityCohorts(cohort_code=value.cohort_code, cohort_status=cohort_status_name)
+                )
+
+                audit_eligibility_cohort_groups.append(
+                    AuditEligibilityCohortGroups(
+                        cohort_code=value.cohort_code, cohort_status=cohort_status_name, cohort_text=value.description
+                    )
+                )
+
+                if value.audit_rules and best_candidate:
+                    if best_candidate.status and best_candidate.status.name == Status.not_eligible.name:
+                        audit_filter_rule = AuditFilterRule(
+                            rule_priority=value.audit_rules[0].rule_priority,
+                            rule_name=value.audit_rules[0].rule_name,
+                        )
+                    if best_candidate.status and best_candidate.status.name == Status.not_actionable.name:
+                        audit_suitability_rule = AuditSuitabilityRule(
+                            rule_priority=value.audit_rules[0].rule_priority,
+                            rule_name=value.audit_rules[0].rule_name,
+                            rule_message=value.audit_rules[0].rule_description,
+                        )
+
+        if best_candidate and best_candidate.status and best_candidate.status.name == Status.actionable.name:
+            audit_redirect_rule = AuditRedirectRule(
+                rule_priority=str(redirect_rule_details[0]), rule_name=redirect_rule_details[1]
+            )
+
+        if suggested_actions is None:
+            audit_actions = None
+        elif len(suggested_actions) > 0:
+            for action in suggested_actions:
+                audit_actions.append(
+                    AuditAction(
+                        internal_action_code=action.internal_action_code,
+                        action_code=action.action_code,
+                        action_type=action.action_type,
+                        action_description=action.action_description,
+                        action_url=str(action.url_link) if action.url_link else None,
+                        action_url_label=action.url_label,
+                    )
+                )
+
+        audit_condition = AuditCondition(
+            campaign_id=campaign_details[0],
+            campaign_version=campaign_details[1],
+            iteration_id=best_active_iteration.id if best_active_iteration else None,
+            iteration_version=best_active_iteration.version if best_active_iteration else None,
+            condition_name=condition_name,
+            status=best_candidate.status.name if best_candidate and best_candidate.status else None,
+            status_text=best_candidate.status.name if best_candidate and best_candidate.status else None,
+            eligibility_cohorts=audit_eligibility_cohorts,
+            eligibility_cohort_groups=audit_eligibility_cohort_groups,
+            filter_rules=audit_filter_rule,
+            suitability_rules=audit_suitability_rule,
+            action_rule=audit_redirect_rule,
+            actions=audit_actions,
+        )
+
+        g.audit_log.response.condition.append(audit_condition)
+
+    @staticmethod
+    def add_response_details(response_id: UUID, last_updated: datetime) -> None:
+        g.audit_log.response.response_id = response_id
+        g.audit_log.response.last_updated = last_updated
+
+    @staticmethod
+    def write_to_firehose(service: AuditService) -> None:
+        service.audit(g.audit_log.model_dump(by_alias=True))

src/eligibility_signposting_api/audit/audit_context.py

@@ -0,0 +1,95 @@
+from datetime import UTC, datetime
+from uuid import UUID
+
+from pydantic import BaseModel, ConfigDict, Field
+from pydantic.alias_generators import to_camel
+
+
+class CamelCaseBaseModel(BaseModel):
+    model_config = ConfigDict(
+        alias_generator=to_camel,
+        populate_by_name=True,
+    )
+
+
+class RequestAuditHeader(CamelCaseBaseModel):
+    x_request_id: str | None = None
+    x_correlation_id: str | None = None
+    nhsd_end_user_organisation_ods: str | None = None
+    nhsd_application_id: str | None = None
+
+
+class RequestAuditQueryParams(CamelCaseBaseModel):
+    category: str | None = None
+    conditions: str | None = None
+    include_actions: str | None = None
+
+
+class RequestAuditData(CamelCaseBaseModel):
+    request_timestamp: datetime = Field(default_factory=lambda: datetime.now(UTC))
+    headers: RequestAuditHeader = Field(default_factory=RequestAuditHeader)
+    query_params: RequestAuditQueryParams = Field(default_factory=RequestAuditQueryParams)
+    nhs_number: str | None = None
+
+
+class AuditEligibilityCohorts(CamelCaseBaseModel):
+    cohort_code: str | None = None
+    cohort_status: str | None = None
+
+
+class AuditEligibilityCohortGroups(CamelCaseBaseModel):
+    cohort_code: str | None = None
+    cohort_text: str | None = None
+    cohort_status: str | None = None
+
+
+class AuditFilterRule(CamelCaseBaseModel):
+    rule_priority: str | None = None
+    rule_name: str | None = None
+
+
+class AuditSuitabilityRule(CamelCaseBaseModel):
+    rule_priority: str | None = None
+    rule_name: str | None = None
+    rule_message: str | None = None
+
+
+class AuditRedirectRule(CamelCaseBaseModel):
+    rule_priority: str | None = None
+    rule_name: str | None = None
+
+
+class AuditAction(CamelCaseBaseModel):
+    internal_action_code: str | None = None
+    action_type: str | None = None
+    action_code: str | None = None
+    action_description: str | None = None
+    action_url: str | None = None
+    action_url_label: str | None = None
+
+
+class AuditCondition(CamelCaseBaseModel):
+    campaign_id: str | None = None
+    campaign_version: str | None = None
+    iteration_id: str | None = None
+    iteration_version: str | None = None
+    condition_name: str | None = None
+    status: str | None = None
+    status_text: str | None = None
+    eligibility_cohorts: list[AuditEligibilityCohorts] | None = None
+    eligibility_cohort_groups: list[AuditEligibilityCohortGroups] | None = None
+    filter_rules: AuditFilterRule | None = None
+    suitability_rules: AuditSuitabilityRule | None = None
+    action_rule: AuditRedirectRule | None = None
+    actions: list[AuditAction] | None = Field(default_factory=list)
+
+
+class ResponseAuditData(CamelCaseBaseModel):
+    response_id: UUID | None = None
+    last_updated: str | None = None
+    condition: list[AuditCondition] = Field(default_factory=list)
+
+
+class AuditEvent(CamelCaseBaseModel):
+    request: RequestAuditData = Field(default_factory=RequestAuditData)
+    response: ResponseAuditData = Field(default_factory=ResponseAuditData)

src/eligibility_signposting_api/audit/audit_models.py

@@ -31,8 +31,9 @@ def audit(self, audit_record: dict) -> None:
         Returns:
             str: The Firehose record ID.
         """
+        data = json.dumps(audit_record, default=str)
         response = self.firehose.put_record(
             DeliveryStreamName=self.audit_delivery_stream,
-            Record={"Data": (json.dumps(audit_record) + "\n").encode("utf-8")},
+            Record={"Data": (data + "\n").encode("utf-8")},
         )
         logger.info("Successfully sent to the Firehose", extra={"firehose_record_id": response["RecordId"]})

…ignposting_api/services/audit_service.py …y_signposting_api/audit/audit_service.pysrc/eligibility_signposting_api/services/audit_service.py renamed to src/eligibility_signposting_api/audit/audit_service.py src/eligibility_signposting_api/services/audit_service.py renamed to src/eligibility_signposting_api/audit/audit_service.py

@@ -15,7 +15,9 @@
 
 RuleName = NewType("RuleName", str)
 RuleDescription = NewType("RuleDescription", str)
+RulePriority = NewType("RulePriority", str)
 
+InternalActionCode = NewType("InternalActionCode", str)
 ActionType = NewType("ActionType", str)
 ActionCode = NewType("ActionCode", str)
 ActionDescription = NewType("ActionDescription", str)
@@ -68,6 +70,7 @@ def best(*statuses: Status) -> Status:
 class Reason:
     rule_type: RuleType
     rule_name: RuleName
+    rule_priority: RulePriority
     rule_description: RuleDescription | None
     matcher_matched: bool
 
@@ -79,6 +82,7 @@ class SuggestedAction:
     action_description: ActionDescription | None
     url_link: UrlLink | None
     url_label: UrlLabel | None
+    internal_action_code: InternalActionCode | None = None
 
 
 @dataclass
@@ -95,6 +99,7 @@ class CohortGroupResult:
     status: Status
     reasons: list[Reason]
     description: str | None
+    audit_rules: list[Reason]
 
 
 @dataclass