New: [AEA-5671] - Trigger Document Ingestion on S3 Upload (#20)

## Summary


🎫 [AEA-5671](https://nhsd-jira.digital.nhs.uk/browse/AEA-5671) Trigger
Document Ingestion on S3 Upload

- ✨ New Feature

### Details
This pull request is to implement a mechanism that listens for new
document uploads to an S3 bucket and triggers a Lambda function in
response. The Lambda function is responsible for initiating the
knowledge base ingestion process.

Key changes include:

- Setup of S3 event notifications for ObjectCreated events.
- Creation of a Lambda function to handle the trigger and call the
ingestion service.
- IAM role and permission updates to enable S3 → Lambda → ingestion
flow.
- Basic error handling and CloudWatch logging for observability.
- Initial test validation to ensure end-to-end triggering and ingestion.

Author: kris-szlapa

.github/workflows/cdk_package_code.yml

@@ -63,6 +63,7 @@ jobs:
         run: |
           pip3 install -r packages/slackBotFunction/requirements.txt -t packages/slackBotFunction
           pip3 install -r packages/createIndexFunction/requirements.txt -t packages/createIndexFunction
+          pip3 install -r packages/syncKnowledgeBaseFunction/requirements.txt -t packages/syncKnowledgeBaseFunction
 
       - name: 'Tar files'
         run: |

.vscode/eps-assist-me.code-workspace

@@ -15,6 +15,10 @@
 		{
 			"name": "packages/slackBotFunction",
 			"path": "../packages/slackBotFunction"
+		},
+		{
+			"name": "packages/syncKnowledgeBaseFunction",
+			"path": "../packages/syncKnowledgeBaseFunction"
 		}
 	],
 	"settings": {

Makefile

@@ -10,8 +10,9 @@ install: install-python install-hooks install-node
 
 install-python:
 	poetry install
-	cd packages/slackBotFunction && pip install -r requirements.txt && pip install -r requirements-test.txt
 	cd packages/createIndexFunction && pip install -r requirements.txt && pip install -r requirements-test.txt
+	cd packages/slackBotFunction && pip install -r requirements.txt && pip install -r requirements-test.txt
+	cd packages/syncKnowledgeBaseFunction && pip install -r requirements.txt && pip install -r requirements-test.txt
 
 install-hooks: install-python
 	poetry run pre-commit install --install-hooks --overwrite
@@ -45,16 +46,19 @@ lint-flake8:
 	poetry run flake8 .
 
 test:
-	cd packages/slackBotFunction && PYTHONPATH=. COVERAGE_FILE=coverage/.coverage python -m pytest
 	cd packages/createIndexFunction && PYTHONPATH=. COVERAGE_FILE=coverage/.coverage python -m pytest
+	cd packages/slackBotFunction && PYTHONPATH=. COVERAGE_FILE=coverage/.coverage python -m pytest
+	cd packages/syncKnowledgeBaseFunction && PYTHONPATH=. COVERAGE_FILE=coverage/.coverage python -m pytest
 
 clean:
 	rm -rf packages/cdk/coverage
 	rm -rf packages/cdk/lib
-	rm -rf packages/slackBotFunction/coverage
-	rm -rf packages/slackBotFunction/.coverage
 	rm -rf packages/createIndexFunction/coverage
 	rm -rf packages/createIndexFunction/.coverage
+	rm -rf packages/slackBotFunction/coverage
+	rm -rf packages/slackBotFunction/.coverage
+	rm -rf packages/syncKnowledgeBaseFunction/coverage
+	rm -rf packages/syncKnowledgeBaseFunction/.coverage
 	rm -rf cdk.out
 	rm -rf .build
 
@@ -107,6 +111,7 @@ cdk-synth:
 		--context logRetentionInDays=30 \
 		--context slackBotToken=dummy \
 		--context slackSigningSecret=dummy
+	./scripts/fix_cfn_guard.sh
 
 cdk-diff:
 	npx cdk diff \

README.md

@@ -9,9 +9,10 @@ The solution consists of:
 
 - **Slack Bot Function**: AWS Lambda function that handles Slack slash commands and integrates with Amazon Bedrock Knowledge Base
 - **Create Index Function**: AWS Lambda function that creates and manages OpenSearch vector indices for the knowledge base
+- **Sync Knowledge Base Function**: AWS Lambda function that automatically triggers knowledge base ingestion when documents are uploaded to S3
 - **OpenSearch Serverless**: Vector database for storing and searching document embeddings
 - **Amazon Bedrock Knowledge Base**: RAG (Retrieval-Augmented Generation) service with guardrails
-- **S3 Storage**: Document storage for the knowledge base
+- **S3 Storage**: Document storage for the knowledge base with automatic sync triggers
 - **AWS CDK**: Infrastructure as Code for deployment
 
 ## Project Structure
@@ -20,13 +21,29 @@ This is a monorepo with the following structure:
 
 ```
 packages/
-├── cdk/                   # AWS CDK infrastructure code
-│   ├── bin/               # CDK app entry point
-│   ├── constructs/        # Reusable CDK constructs
-│   ├── resources/         # AWS resource definitions
-│   └── stacks/            # CDK stack definitions
-├── createIndexFunction/   # Lambda function for OpenSearch index management
-└── slackBotFunction/      # Lambda function for Slack bot integration
+├── cdk/                        # AWS CDK infrastructure code
+│   ├── bin/                    # CDK app entry point
+│   │   └── utils/              # CDK utility functions
+│   ├── constructs/             # Reusable CDK constructs
+│   │   └── RestApiGateway/     # API Gateway specific constructs
+│   ├── resources/              # AWS resource definitions
+│   └── stacks/                 # CDK stack definitions
+├── createIndexFunction/        # Lambda function for OpenSearch index management
+│   ├── app/                    # Application code
+│   │   ├── config/             # Configuration and environment variables
+│   │   └── handler.py          # Lambda handler
+│   └── tests/                  # Unit tests
+├── slackBotFunction/           # Lambda function for Slack bot integration
+│   ├── app/                    # Application code
+│   │   ├── config/             # Configuration and environment variables
+│   │   ├── slack/              # Slack-specific logic
+│   │   └── handler.py          # Lambda handler
+│   └── tests/                  # Unit tests
+└── syncKnowledgeBaseFunction/  # Lambda function for automatic knowledge base sync
+    ├── app/                    # Application code
+    │   ├── config/             # Configuration and environment variables
+    │   └── handler.py          # Lambda handler
+    └── tests/                  # Unit tests
 ```
 
 ## Contributing
@@ -149,14 +166,15 @@ These are used to do common commands related to cdk
 
 #### Linting and testing
 
-- `lint` Runs lint for GitHub Actions and scripts.
+- `lint` Runs all linting checks
 - `lint-black` Runs black formatter on Python code.
 - `lint-flake8` Runs flake8 linter on Python code.
 - `lint-githubactions` Lints the repository's GitHub Actions workflows.
 - `lint-githubaction-scripts` Lints all shell scripts in `.github/scripts` using ShellCheck.
-- `test` Runs unit tests for Lambda functions.
 - `cfn-guard` Runs cfn-guard against CDK resources.
+- `git-secrets-docker-setup` Sets up git-secrets Docker container.
 - `pre-commit` Runs pre-commit hooks on all files.
+- `test` Runs unit tests for Lambda functions.
 
 #### Compiling
 

package.json

@@ -14,9 +14,7 @@
   "author": "NHS Digital",
   "license": "MIT",
   "workspaces": [
-    "packages/cdk",
-    "packages/querytool",
-    "packages/slackbot"
+    "packages/cdk"
   ],
   "devDependencies": {
     "@semantic-release/changelog": "^6.0.3",

packages/cdk/constructs/LambdaFunction.ts

@@ -132,7 +132,7 @@ export class LambdaFunction extends Construct {
       memorySize: 256,
       timeout: Duration.seconds(50),
       architecture: Architecture.X86_64,
-      handler: "app.handler",
+      handler: "app.handler.handler",
       code: Code.fromAsset(props.packageBasePath),
       role,
       environment: {

packages/cdk/constructs/OpenSearchCollection.ts

@@ -4,15 +4,26 @@ import {CfnCollection, CfnSecurityPolicy, CfnAccessPolicy} from "aws-cdk-lib/aws
 export interface OpenSearchCollectionProps {
   readonly collectionName: string
   readonly principals: Array<string>
+  readonly region: string
+  readonly account: string
 }
 
 export class OpenSearchCollection extends Construct {
   public readonly collection: CfnCollection
   public readonly endpoint: string
+  private readonly region: string
+  private readonly account: string
+
+  public get collectionArn(): string {
+    return `arn:aws:aoss:${this.region}:${this.account}:collection/${this.collection.attrId}`
+  }
 
   constructor(scope: Construct, id: string, props: OpenSearchCollectionProps) {
     super(scope, id)
 
+    this.region = props.region
+    this.account = props.account
+
     // Encryption policy using AWS-managed keys
     const encryptionPolicy = new CfnSecurityPolicy(this, "EncryptionPolicy", {
       name: `${props.collectionName}-encryption`,

packages/cdk/constructs/S3LambdaNotification.ts

@@ -0,0 +1,36 @@
+import {Construct} from "constructs"
+import {Bucket, EventType} from "aws-cdk-lib/aws-s3"
+import {LambdaDestination} from "aws-cdk-lib/aws-s3-notifications"
+import {Function as LambdaFunction} from "aws-cdk-lib/aws-lambda"
+
+export interface S3LambdaNotificationProps {
+  bucket: Bucket
+  lambdaFunction: LambdaFunction
+}
+
+export class S3LambdaNotification extends Construct {
+  constructor(scope: Construct, id: string, props: S3LambdaNotificationProps) {
+    super(scope, id)
+
+    const lambdaDestination = new LambdaDestination(props.lambdaFunction)
+
+    // Trigger knowledge base sync only for supported document types
+    const supportedExtensions = [".pdf", ".txt", ".md", ".csv", ".doc", ".docx", ".xls", ".xlsx", ".html", ".json"]
+
+    supportedExtensions.forEach(ext => {
+      // Handle all file creation/modification events
+      props.bucket.addEventNotification(
+        EventType.OBJECT_CREATED,
+        lambdaDestination,
+        {suffix: ext}
+      )
+
+      // Handle all file deletion events
+      props.bucket.addEventNotification(
+        EventType.OBJECT_REMOVED,
+        lambdaDestination,
+        {suffix: ext}
+      )
+    })
+  }
+}

packages/cdk/nagSuppressions.ts

@@ -35,6 +35,21 @@ export const nagSuppressions = (stack: Stack) => {
     ]
   )
 
+  // Suppress wildcard log permissions for SyncKnowledgeBase Lambda
+  safeAddNagSuppression(
+    stack,
+    "/EpsAssistMeStack/Functions/SyncKnowledgeBaseFunction/LambdaPutLogsManagedPolicy/Resource",
+    [
+      {
+        id: "AwsSolutions-IAM5",
+        reason: "Wildcard permissions are required for log stream access under known paths.",
+        appliesTo: [
+          "Resource::<FunctionsSyncKnowledgeBaseFunctionLambdaLogGroupB19BE2BE.Arn>:log-stream:*"
+        ]
+      }
+    ]
+  )
+
   // Suppress API Gateway validation warning for Apis construct
   safeAddNagSuppression(
     stack,
@@ -87,18 +102,18 @@ export const nagSuppressions = (stack: Stack) => {
     ]
   )
 
-  // Suppress IAM wildcard permissions for Bedrock execution managed policy
+  // Suppress IAM wildcard permissions for Bedrock execution role policy
   safeAddNagSuppression(
     stack,
-    "/EpsAssistMeStack/IamResources/BedrockExecutionManagedPolicy/Resource",
+    "/EpsAssistMeStack/BedrockExecutionRole/Policy/Resource",
     [
       {
         id: "AwsSolutions-IAM5",
         reason: "Bedrock Knowledge Base requires these permissions to access S3 documents and OpenSearch collection.",
         appliesTo: [
-          "Action::bedrock:Delete*",
           "Resource::<StorageDocsBucketepsamDocsF25F63F1.Arn>/*",
-          "Resource::<StorageDocsBucketepsampr16Docs240CC945.Arn>/*",
+          "Resource::<StorageDocsBucketepsampr20Docs075F648F.Arn>/*",
+          "Action::bedrock:Delete*",
           `Resource::arn:aws:bedrock:eu-west-2:${account}:knowledge-base/*`,
           `Resource::arn:aws:aoss:eu-west-2:${account}:collection/*`,
           "Resource::*"
@@ -107,10 +122,10 @@ export const nagSuppressions = (stack: Stack) => {
     ]
   )
 
-  // Suppress wildcard permissions for CreateIndex managed policy
+  // Suppress wildcard permissions for CreateIndex policy
   safeAddNagSuppression(
     stack,
-    "/EpsAssistMeStack/IamResources/CreateIndexManagedPolicy/Resource",
+    "/EpsAssistMeStack/RuntimePolicies/CreateIndexPolicy/Resource",
     [
       {
         id: "AwsSolutions-IAM5",
@@ -123,18 +138,16 @@ export const nagSuppressions = (stack: Stack) => {
     ]
   )
 
-  // Suppress wildcard permissions for SlackBot managed policy
+  // Suppress wildcard permissions for SlackBot policy
   safeAddNagSuppression(
     stack,
-    "/EpsAssistMeStack/IamResources/SlackBotManagedPolicy/Resource",
+    "/EpsAssistMeStack/RuntimePolicies/SlackBotPolicy/Resource",
     [
       {
         id: "AwsSolutions-IAM5",
-        reason: "SlackBot Lambda needs access to all guardrails, knowledge bases, and functions for content filtering and self-invocation.",
+        reason: "SlackBot Lambda needs wildcard access for Lambda functions (self-invocation) and KMS operations.",
         appliesTo: [
           `Resource::arn:aws:lambda:eu-west-2:${account}:function:*`,
-          `Resource::arn:aws:bedrock:eu-west-2:${account}:guardrail/*`,
-          `Resource::arn:aws:bedrock:eu-west-2:${account}:knowledge-base/*`,
           "Action::kms:GenerateDataKey*",
           "Action::kms:ReEncrypt*"
         ]
@@ -177,6 +190,40 @@ export const nagSuppressions = (stack: Stack) => {
     ]
   )
 
+  // Suppress AWS managed policy usage in BucketNotificationsHandler (wildcard for any hash)
+  const bucketNotificationHandlers = stack.node.findAll().filter(node =>
+    node.node.id.startsWith("BucketNotificationsHandler")
+  )
+
+  bucketNotificationHandlers.forEach(handler => {
+    safeAddNagSuppression(
+      stack,
+      `${handler.node.path}/Role/Resource`,
+      [
+        {
+          id: "AwsSolutions-IAM4",
+          reason: "Auto-generated CDK role uses AWS managed policy for basic Lambda execution.",
+          appliesTo: [
+            "Policy::arn:<AWS::Partition>:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
+          ]
+        }
+      ]
+    )
+
+    safeAddNagSuppression(
+      stack,
+      `${handler.node.path}/Role/DefaultPolicy/Resource`,
+      [
+        {
+          id: "AwsSolutions-IAM5",
+          reason: "Auto-generated CDK role requires wildcard permissions for S3 bucket notifications.",
+          appliesTo: [
+            "Resource::*"
+          ]
+        }
+      ]
+    )
+  })
 }
 
 const safeAddNagSuppression = (stack: Stack, path: string, suppressions: Array<NagPackSuppression>) => {