New: [AEA-5546] - Use prompts for better answers (#27)

## Summary

🎫 [AEA-5546](https://nhsd-jira.digital.nhs.uk/browse/AEA-5546) Use
prompts for better answers

:sparkles: New Feature
:robot: Operational or Infrastructure Change

### Details

This change improves the relevance of Slack bot answers by introducing a
prompt-building and query-rewriting layer before sending user questions
to Bedrock’s Knowledge Base retrieve_and_generate API.

Key updates:

- Added system prompt to instruct Bedrock on response style, source
restrictions, and EPS/NHS domain behaviour.
- Implemented query rewriting to:
  - Strip Slack-specific noise (mentions, emojis, formatting).
  - Expand EPS/NHS domain acronyms and terminology.
  - Normalize and clarify vague user input.
- Modified get_bedrock_knowledgebase_response to send the prompt +
rewritten query instead of the raw Slack message.
- Externalised prompt content for easier iteration without code changes.
- Added logging to compare original vs rewritten queries and help with
post-deployment tuning.

Outcome: Bedrock receives clearer, more context-rich queries, resulting
in answers that are more relevant and better grounded in the knowledge
base.

Author: kris-szlapa

.flake8

@@ -2,4 +2,4 @@
 max-line-length = 120
 exclude = .*,venv,node_modules,cdk.out
 max-complexity = 10
-ignore = F821,E203
+ignore = F821,E203,W503

README.md

@@ -36,6 +36,7 @@ packages/
 ├── slackBotFunction/           # Lambda function for Slack bot integration
 │   ├── app/                    # Application code
 │   │   ├── config/             # Configuration and environment variables
+│   │   ├── services/           # Business logic services
 │   │   ├── slack/              # Slack-specific logic
 │   │   └── handler.py          # Lambda handler
 │   └── tests/                  # Unit tests
@@ -196,7 +197,7 @@ These are used to do common commands related to cdk
 This .github folder contains workflows and templates related to GitHub, along with actions and scripts pertaining to Jira.
 
 - `dependabot.yml` Dependabot definition file.
-- `pull_request_template.yml` Template for pull requests.
+- `pull_request_template.md` Template for pull requests.
 
 Actions are in the `.github/actions` folder:
 
@@ -216,10 +217,11 @@ Scripts are in the `.github/scripts` folder:
 Workflows are in the `.github/workflows` folder:
 
 - `combine-dependabot-prs.yml` Workflow for combining dependabot pull requests. Runs on demand.
+- `create_release_notes.yml` Generates release notes for deployments and environment updates.
 - `delete_old_cloudformation_stacks.yml` Workflow for deleting old cloud formation stacks. Runs daily.
 - `dependabot_auto_approve_and_merge.yml` Workflow to auto merge dependabot updates.
 - `pr_title_check.yml` Checks PR titles for required prefix and ticket or dependabot reference.
-- `pr-link.yaml` This workflow template links Pull Requests to Jira tickets and runs when a pull request is opened.
+- `pr-link.yml` This workflow template links Pull Requests to Jira tickets and runs when a pull request is opened.
 - `pull_request.yml` Called when pull request is opened or updated. Packages and deploys the code to dev AWS account for testing.
 - `release.yml` Runs on demand to create a release and deploy to all environments.
 - `cdk_package_code.yml` Packages code into a docker image and uploads to a github artifact for later deployment.

packages/cdk/constructs/BedrockPrompt.ts

@@ -0,0 +1,36 @@
+import {Construct} from "constructs"
+import {CfnPrompt} from "aws-cdk-lib/aws-bedrock"
+
+export interface BedrockPromptProps {
+  promptName: string
+  promptText: string
+  description: string
+}
+
+export class BedrockPrompt extends Construct {
+  public readonly promptArn: string
+  public readonly promptName: string
+
+  constructor(scope: Construct, id: string, props: BedrockPromptProps) {
+    super(scope, id)
+
+    const prompt = new CfnPrompt(this, "Prompt", {
+      name: props.promptName,
+      description: props.description,
+      variants: [
+        {
+          name: "default",
+          templateType: "TEXT",
+          templateConfiguration: {
+            text: {
+              text: props.promptText
+            }
+          }
+        }
+      ]
+    })
+
+    this.promptArn = prompt.attrArn
+    this.promptName = prompt.name
+  }
+}

packages/cdk/nagSuppressions.ts

@@ -35,10 +35,7 @@ export const nagSuppressions = (stack: Stack) => {
     [
       {
         id: "AwsSolutions-IAM5",
-        reason: "Wildcard permissions are required for log stream access under known paths.",
-        appliesTo: [
-          "Resource::<FunctionsSyncKnowledgeBaseFunctionLambdaLogGroupB19BE2BE.Arn>:log-stream:*"
-        ]
+        reason: "Wildcard permissions are required for log stream access under known paths."
       }
     ]
   )
@@ -181,10 +178,7 @@ export const nagSuppressions = (stack: Stack) => {
       [
         {
           id: "AwsSolutions-IAM4",
-          reason: "Auto-generated CDK role uses AWS managed policy for basic Lambda execution.",
-          appliesTo: [
-            "Policy::arn:<AWS::Partition>:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
-          ]
+          reason: "Auto-generated CDK role uses AWS managed policy for basic Lambda execution."
         }
       ]
     )
@@ -195,10 +189,7 @@ export const nagSuppressions = (stack: Stack) => {
       [
         {
           id: "AwsSolutions-IAM5",
-          reason: "Auto-generated CDK role requires wildcard permissions for S3 bucket notifications.",
-          appliesTo: [
-            "Resource::*"
-          ]
+          reason: "Auto-generated CDK role requires wildcard permissions for S3 bucket notifications."
         }
       ]
     )
@@ -213,9 +204,6 @@ const safeAddNagSuppression = (stack: Stack, path: string, suppressions: Array<N
   }
 }
 
-// Apply the same nag suppression to multiple resources
 const safeAddNagSuppressionGroup = (stack: Stack, paths: Array<string>, suppressions: Array<NagPackSuppression>) => {
-  for (const p of paths) {
-    safeAddNagSuppression(stack, p, suppressions)
-  }
+  paths.forEach(path => safeAddNagSuppression(stack, path, suppressions))
 }

packages/cdk/resources/BedrockPromptResources.ts

@@ -0,0 +1,24 @@
+import {Construct} from "constructs"
+import {BedrockPrompt} from "../constructs/BedrockPrompt"
+
+export interface BedrockPromptResourcesProps {
+  readonly stackName: string
+}
+
+export class BedrockPromptResources extends Construct {
+  public readonly queryReformulationPrompt: BedrockPrompt
+
+  constructor(scope: Construct, id: string, props: BedrockPromptResourcesProps) {
+    super(scope, id)
+
+    this.queryReformulationPrompt = new BedrockPrompt(this, "QueryReformulationPrompt", {
+      promptName: `${props.stackName}-queryReformulation`,
+      promptText: `Return the user query exactly as provided without any modifications, changes, or reformulations.
+Do not alter, rephrase, or modify the input in any way.
+Simply return: {{user_query}}
+
+User Query: {{user_query}}`,
+      description: "Prompt for reformulating user queries to improve RAG retrieval"
+    })
+  }
+}

packages/cdk/resources/Functions.ts

@@ -7,6 +7,9 @@ import {TableV2} from "aws-cdk-lib/aws-dynamodb"
 
 // Claude model for RAG responses
 const RAG_MODEL_ID = "anthropic.claude-3-sonnet-20240229-v1:0"
+// Claude model for query reformulation
+const QUERY_REFORMULATION_MODEL_ID = "anthropic.claude-3-haiku-20240307-v1:0"
+const QUERY_REFORMULATION_PROMPT_VERSION = "DRAFT"
 const BEDROCK_KB_DATA_SOURCE = "eps-assist-kb-ds"
 const LAMBDA_MEMORY_SIZE = "265"
 
@@ -31,6 +34,7 @@ export interface FunctionsProps {
   readonly slackBotTokenSecret: Secret
   readonly slackBotSigningSecret: Secret
   readonly slackBotStateTable: TableV2
+  readonly promptName: string
 }
 
 export class Functions extends Construct {
@@ -62,14 +66,17 @@ export class Functions extends Construct {
       additionalPolicies: [props.slackBotManagedPolicy],
       environmentVariables: {
         "RAG_MODEL_ID": RAG_MODEL_ID,
+        "QUERY_REFORMULATION_MODEL_ID": QUERY_REFORMULATION_MODEL_ID,
         "KNOWLEDGEBASE_ID": props.knowledgeBaseId,
         "BEDROCK_KB_DATA_SOURCE": BEDROCK_KB_DATA_SOURCE,
         "LAMBDA_MEMORY_SIZE": LAMBDA_MEMORY_SIZE,
         "SLACK_BOT_TOKEN_PARAMETER": props.slackBotTokenParameter.parameterName,
         "SLACK_SIGNING_SECRET_PARAMETER": props.slackSigningSecretParameter.parameterName,
         "GUARD_RAIL_ID": props.guardrailId,
         "GUARD_RAIL_VERSION": props.guardrailVersion,
-        "SLACK_BOT_STATE_TABLE": props.slackBotStateTable.tableName
+        "SLACK_BOT_STATE_TABLE": props.slackBotStateTable.tableName,
+        "QUERY_REFORMULATION_PROMPT_NAME": props.promptName,
+        "QUERY_REFORMULATION_PROMPT_VERSION": QUERY_REFORMULATION_PROMPT_VERSION
       }
     })
 

packages/cdk/resources/RuntimePolicies.ts

@@ -3,6 +3,8 @@ import {PolicyStatement, ManagedPolicy} from "aws-cdk-lib/aws-iam"
 
 // Claude model for RAG responses
 const RAG_MODEL_ID = "anthropic.claude-3-sonnet-20240229-v1:0"
+// Claude model for query reformulation
+const QUERY_REFORMULATION_MODEL_ID = "anthropic.claude-3-haiku-20240307-v1:0"
 
 export interface RuntimePoliciesProps {
   readonly region: string
@@ -14,6 +16,7 @@ export interface RuntimePoliciesProps {
   readonly knowledgeBaseArn: string
   readonly guardrailArn: string
   readonly dataSourceArn: string
+  readonly promptName: string
 }
 
 export class RuntimePolicies extends Construct {
@@ -53,7 +56,35 @@ export class RuntimePolicies extends Construct {
     // Create managed policy for SlackBot Lambda function
     const slackBotPolicy = new PolicyStatement({
       actions: ["bedrock:InvokeModel"],
-      resources: [`arn:aws:bedrock:${props.region}::foundation-model/${RAG_MODEL_ID}`]
+      resources: [
+        `arn:aws:bedrock:${props.region}::foundation-model/${RAG_MODEL_ID}`,
+        `arn:aws:bedrock:${props.region}::foundation-model/${QUERY_REFORMULATION_MODEL_ID}`
+      ]
+    })
+
+    // Compehensive Bedrock prompt policy - includes all prompt management permissions
+    const slackBotPromptPolicy = new PolicyStatement({
+      sid: "PromptManagementPermissions",
+      actions: [
+        "bedrock:CreatePrompt",
+        "bedrock:UpdatePrompt",
+        "bedrock:GetPrompt",
+        "bedrock:ListPrompts",
+        "bedrock:DeletePrompt",
+        "bedrock:CreatePromptVersion",
+        "bedrock:OptimizePrompt",
+        "bedrock:GetFoundationModel",
+        "bedrock:ListFoundationModels",
+        "bedrock:GetInferenceProfile",
+        "bedrock:ListInferenceProfiles",
+        "bedrock:InvokeModel",
+        "bedrock:InvokeModelWithResponseStream",
+        "bedrock:RenderPrompt",
+        "bedrock:TagResource",
+        "bedrock:UntagResource",
+        "bedrock:ListTagsForResource"
+      ],
+      resources: ["*"] // Use wildcard as recommended by AWS docs
     })
 
     const slackBotKnowledgeBasePolicy = new PolicyStatement({
@@ -108,6 +139,7 @@ export class RuntimePolicies extends Construct {
       description: "Policy for SlackBot Lambda to access Bedrock, SSM, Lambda, DynamoDB, and KMS",
       statements: [
         slackBotPolicy,
+        slackBotPromptPolicy,
         slackBotKnowledgeBasePolicy,
         slackBotSSMPolicy,
         slackBotLambdaPolicy,

packages/cdk/stacks/EpsAssistMeStack.ts

@@ -15,6 +15,7 @@ import {BedrockExecutionRole} from "../resources/BedrockExecutionRole"
 import {RuntimePolicies} from "../resources/RuntimePolicies"
 import {VectorIndex} from "../resources/VectorIndex"
 import {DatabaseTables} from "../resources/DatabaseTables"
+import {BedrockPromptResources} from "../resources/BedrockPromptResources"
 import {S3LambdaNotification} from "../constructs/S3LambdaNotification"
 
 const VECTOR_INDEX_NAME = "eps-assist-os-index"
@@ -55,6 +56,11 @@ export class EpsAssistMeStack extends Stack {
       stackName: props.stackName
     })
 
+    // Create Bedrock Prompt Resources
+    const bedrockPromptResources = new BedrockPromptResources(this, "BedrockPromptResources", {
+      stackName: props.stackName
+    })
+
     // Create Storage construct first as it has no dependencies
     const storage = new Storage(this, "Storage", {
       stackName: props.stackName
@@ -88,7 +94,7 @@ export class EpsAssistMeStack extends Stack {
       account
     })
 
-    // Create runtime policies that depend on VectorKB ARNs
+    // Create runtime policies with resource dependencies
     const runtimePolicies = new RuntimePolicies(this, "RuntimePolicies", {
       region,
       account,
@@ -98,7 +104,8 @@ export class EpsAssistMeStack extends Stack {
       slackBotStateTableKmsKeyArn: tables.slackBotStateTable.kmsKey.keyArn,
       knowledgeBaseArn: vectorKB.knowledgeBase.attrKnowledgeBaseArn,
       guardrailArn: vectorKB.guardrail.attrGuardrailArn,
-      dataSourceArn: vectorKB.dataSourceArn
+      dataSourceArn: vectorKB.dataSourceArn,
+      promptName: bedrockPromptResources.queryReformulationPrompt.promptName
     })
 
     // Create Functions construct with actual values from VectorKB
@@ -122,7 +129,8 @@ export class EpsAssistMeStack extends Stack {
       account,
       slackBotTokenSecret: secrets.slackBotTokenSecret,
       slackBotSigningSecret: secrets.slackBotSigningSecret,
-      slackBotStateTable: tables.slackBotStateTable.table
+      slackBotStateTable: tables.slackBotStateTable.table,
+      promptName: bedrockPromptResources.queryReformulationPrompt.promptName
     })
 
     // Create vector index after Functions are created
@@ -158,6 +166,12 @@ export class EpsAssistMeStack extends Stack {
       description: "Slack Events API endpoint for @mentions and direct messages"
     })
 
+    // Output: Bedrock Prompt ARN
+    new CfnOutput(this, "QueryReformulationPromptArn", {
+      value: bedrockPromptResources.queryReformulationPrompt.promptArn,
+      description: "ARN of the query reformulation prompt in Bedrock"
+    })
+
     // Final CDK Nag Suppressions
     nagSuppressions(this)
   }

packages/createIndexFunction/app/handler.py

@@ -10,11 +10,8 @@
 import time
 import boto3
 from opensearchpy import OpenSearch, RequestsHttpConnection, AWSV4SignerAuth
-from aws_lambda_powertools import Logger
 from aws_lambda_powertools.utilities.typing import LambdaContext
-from app.config.config import AWS_REGION
-
-logger = Logger(service="createIndexFunction")
+from app.config.config import AWS_REGION, logger
 
 
 def get_opensearch_client(endpoint):
@@ -23,7 +20,7 @@ def get_opensearch_client(endpoint):
     """
     # Determine service type: AOSS (Serverless) or ES (managed)
     service = "aoss" if "aoss" in endpoint else "es"
-    logger.debug(f"Connecting to OpenSearch service: {service} at {endpoint}")
+    logger.debug("Connecting to OpenSearch service", extra={"service": service, "endpoint": endpoint})
     return OpenSearch(
         hosts=[{"host": endpoint, "port": 443}],
         http_auth=AWSV4SignerAuth(
@@ -45,22 +42,22 @@ def wait_for_index_aoss(opensearch_client, index_name, timeout=300, poll_interva
     AOSS has eventual consistency, so we need to poll until the index
     is fully created and mappings are available.
     """
-    logger.info(f"Waiting for index '{index_name}' to be available in AOSS...")
+    logger.info("Waiting for index to be available in AOSS", extra={"index_name": index_name})
     start = time.time()
     while True:
         try:
             if opensearch_client.indices.exists(index=index_name):
                 # Verify mappings are also available (not just index existence)
                 mapping = opensearch_client.indices.get_mapping(index=index_name)
                 if mapping and index_name in mapping:
-                    logger.info(f"Index '{index_name}' exists and mappings are ready.")
+                    logger.info("Index exists and mappings are ready", extra={"index_name": index_name})
                     return True
             else:
-                logger.info(f"Index '{index_name}' does not exist yet...")
+                logger.info("Index does not exist yet", extra={"index_name": index_name})
         except Exception as exc:
-            logger.info(f"Still waiting for index '{index_name}': {exc}")
+            logger.info("Still waiting for index", extra={"index_name": index_name, "error": str(exc)})
         if time.time() - start > timeout:
-            logger.error(f"Timed out waiting for index '{index_name}' to be available.")
+            logger.error("Timed out waiting for index to be available", extra={"index_name": index_name})
             return False
         time.sleep(poll_interval)
 
@@ -109,18 +106,18 @@ def create_and_wait_for_index(client, index_name):
 
     try:
         if not client.indices.exists(index=params["index"]):
-            logger.info(f"Creating index {params['index']}")
+            logger.info("Creating index", extra={"index_name": params["index"]})
             client.indices.create(index=params["index"], body=params["body"])
-            logger.info(f"Index {params['index']} creation initiated.")
+            logger.info("Index creation initiated", extra={"index_name": params["index"]})
         else:
-            logger.info(f"Index {params['index']} already exists")
+            logger.info("Index already exists", extra={"index_name": params["index"]})
 
         if not wait_for_index_aoss(client, params["index"]):
             raise RuntimeError(f"Index {params['index']} failed to appear in time")
 
-        logger.info(f"Index {params['index']} is ready and active.")
+        logger.info("Index is ready and active", extra={"index_name": params["index"]})
     except Exception as e:
-        logger.error(f"Error creating or waiting for index: {e}")
+        logger.error("Error creating or waiting for index", extra={"error": str(e)})
         raise e
 
 
@@ -181,10 +178,10 @@ def handler(event: dict, context: LambdaContext) -> dict:
             try:
                 if client.indices.exists(index=index_name):
                     client.indices.delete(index=index_name)
-                    logger.info(f"Deleted index {index_name}")
+                    logger.info("Deleted index", extra={"index_name": index_name})
             except Exception as e:
                 # Don't fail deletion if index cleanup fails
-                logger.error(f"Error deleting index: {e}")
+                logger.error("Error deleting index", extra={"error": str(e)})
             return {
                 "PhysicalResourceId": event.get("PhysicalResourceId", f"index-{index_name}"),
                 "Status": "SUCCESS",
@@ -193,5 +190,5 @@ def handler(event: dict, context: LambdaContext) -> dict:
             raise ValueError(f"Invalid request type: {request_type}")
 
     except Exception as e:
-        logger.error(f"Error processing request: {e}")
+        logger.error("Error processing request", extra={"error": str(e)})
         raise