From b16c33e998e4fc8a1af3baa7c38a260d490315f9 Mon Sep 17 00:00:00 2001 From: Michael Kaiser Date: Fri, 11 Jul 2025 11:51:34 -0500 Subject: [PATCH 01/23] Initial project setup: Add CDK configuration files and project structure --- typescript/postgres-lambda/.gitignore | 8 ++ typescript/postgres-lambda/.npmignore | 6 ++ typescript/postgres-lambda/cdk.json | 98 +++++++++++++++++++++++ typescript/postgres-lambda/jest.config.js | 8 ++ typescript/postgres-lambda/package.json | 30 +++++++ typescript/postgres-lambda/tsconfig.json | 31 +++++++ 6 files changed, 181 insertions(+) create mode 100644 typescript/postgres-lambda/.gitignore create mode 100644 typescript/postgres-lambda/.npmignore create mode 100644 typescript/postgres-lambda/cdk.json create mode 100644 typescript/postgres-lambda/jest.config.js create mode 100644 typescript/postgres-lambda/package.json create mode 100644 typescript/postgres-lambda/tsconfig.json diff --git a/typescript/postgres-lambda/.gitignore b/typescript/postgres-lambda/.gitignore new file mode 100644 index 0000000000..f60797b6a9 --- /dev/null +++ b/typescript/postgres-lambda/.gitignore @@ -0,0 +1,8 @@ +*.js +!jest.config.js +*.d.ts +node_modules + +# CDK asset staging directory +.cdk.staging +cdk.out diff --git a/typescript/postgres-lambda/.npmignore b/typescript/postgres-lambda/.npmignore new file mode 100644 index 0000000000..c1d6d45dcf --- /dev/null +++ b/typescript/postgres-lambda/.npmignore @@ -0,0 +1,6 @@ +*.ts +!*.d.ts + +# CDK asset staging directory +.cdk.staging +cdk.out diff --git a/typescript/postgres-lambda/cdk.json b/typescript/postgres-lambda/cdk.json new file mode 100644 index 0000000000..15397fccbc --- /dev/null +++ b/typescript/postgres-lambda/cdk.json @@ -0,0 +1,98 @@ +{ + "app": "npx ts-node --prefer-ts-exts bin/postgres-lambda.ts", + "watch": { + "include": [ + "**" + ], + "exclude": [ + "README.md", + "cdk*.json", + "**/*.d.ts", + "**/*.js", + "tsconfig.json", + "package*.json", + "yarn.lock", + "node_modules", + "test" + ] + }, + "context": { + "@aws-cdk/aws-lambda:recognizeLayerVersion": true, + "@aws-cdk/core:checkSecretUsage": true, + "@aws-cdk/core:target-partitions": [ + "aws", + "aws-cn" + ], + "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, + "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, + "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true, + "@aws-cdk/aws-iam:minimizePolicies": true, + "@aws-cdk/core:validateSnapshotRemovalPolicy": true, + "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, + "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, + "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true, + "@aws-cdk/aws-apigateway:disableCloudWatchRole": true, + "@aws-cdk/core:enablePartitionLiterals": true, + "@aws-cdk/aws-events:eventsTargetQueueSameAccount": true, + "@aws-cdk/aws-ecs:disableExplicitDeploymentControllerForCircuitBreaker": true, + "@aws-cdk/aws-iam:importedRoleStackSafeDefaultPolicyName": true, + "@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true, + "@aws-cdk/aws-route53-patters:useCertificate": true, + "@aws-cdk/customresources:installLatestAwsSdkDefault": false, + "@aws-cdk/aws-rds:databaseProxyUniqueResourceName": true, + "@aws-cdk/aws-codedeploy:removeAlarmsFromDeploymentGroup": true, + "@aws-cdk/aws-apigateway:authorizerChangeDeploymentLogicalId": true, + "@aws-cdk/aws-ec2:launchTemplateDefaultUserData": true, + "@aws-cdk/aws-secretsmanager:useAttachedSecretResourcePolicyForSecretTargetAttachments": true, + "@aws-cdk/aws-redshift:columnId": true, + "@aws-cdk/aws-stepfunctions-tasks:enableEmrServicePolicyV2": true, + "@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": true, + "@aws-cdk/aws-apigateway:requestValidatorUniqueId": true, + "@aws-cdk/aws-kms:aliasNameRef": true, + "@aws-cdk/aws-kms:applyImportedAliasPermissionsToPrincipal": true, + "@aws-cdk/aws-autoscaling:generateLaunchTemplateInsteadOfLaunchConfig": true, + "@aws-cdk/core:includePrefixInUniqueNameGeneration": true, + "@aws-cdk/aws-efs:denyAnonymousAccess": true, + "@aws-cdk/aws-opensearchservice:enableOpensearchMultiAzWithStandby": true, + "@aws-cdk/aws-lambda-nodejs:useLatestRuntimeVersion": true, + "@aws-cdk/aws-efs:mountTargetOrderInsensitiveLogicalId": true, + "@aws-cdk/aws-rds:auroraClusterChangeScopeOfInstanceParameterGroupWithEachParameters": true, + "@aws-cdk/aws-appsync:useArnForSourceApiAssociationIdentifier": true, + "@aws-cdk/aws-rds:preventRenderingDeprecatedCredentials": true, + "@aws-cdk/aws-codepipeline-actions:useNewDefaultBranchForCodeCommitSource": true, + "@aws-cdk/aws-cloudwatch-actions:changeLambdaPermissionLogicalIdForLambdaAction": true, + "@aws-cdk/aws-codepipeline:crossAccountKeysDefaultValueToFalse": true, + "@aws-cdk/aws-codepipeline:defaultPipelineTypeToV2": true, + "@aws-cdk/aws-kms:reduceCrossAccountRegionPolicyScope": true, + "@aws-cdk/aws-eks:nodegroupNameAttribute": true, + "@aws-cdk/aws-ec2:ebsDefaultGp3Volume": true, + "@aws-cdk/aws-ecs:removeDefaultDeploymentAlarm": true, + "@aws-cdk/custom-resources:logApiResponseDataPropertyTrueDefault": false, + "@aws-cdk/aws-s3:keepNotificationInImportedBucket": false, + "@aws-cdk/aws-ecs:enableImdsBlockingDeprecatedFeature": false, + "@aws-cdk/aws-ecs:disableEcsImdsBlocking": true, + "@aws-cdk/aws-ecs:reduceEc2FargateCloudWatchPermissions": true, + "@aws-cdk/aws-dynamodb:resourcePolicyPerReplica": true, + "@aws-cdk/aws-ec2:ec2SumTImeoutEnabled": true, + "@aws-cdk/aws-appsync:appSyncGraphQLAPIScopeLambdaPermission": true, + "@aws-cdk/aws-rds:setCorrectValueForDatabaseInstanceReadReplicaInstanceResourceId": true, + "@aws-cdk/core:cfnIncludeRejectComplexResourceUpdateCreatePolicyIntrinsics": true, + "@aws-cdk/aws-lambda-nodejs:sdkV3ExcludeSmithyPackages": true, + "@aws-cdk/aws-stepfunctions-tasks:fixRunEcsTaskPolicy": true, + "@aws-cdk/aws-ec2:bastionHostUseAmazonLinux2023ByDefault": true, + "@aws-cdk/aws-route53-targets:userPoolDomainNameMethodWithoutCustomResource": true, + "@aws-cdk/aws-elasticloadbalancingV2:albDualstackWithoutPublicIpv4SecurityGroupRulesDefault": true, + "@aws-cdk/aws-iam:oidcRejectUnauthorizedConnections": true, + "@aws-cdk/core:enableAdditionalMetadataCollection": true, + "@aws-cdk/aws-lambda:createNewPoliciesWithAddToRolePolicy": false, + "@aws-cdk/aws-s3:setUniqueReplicationRoleName": true, + "@aws-cdk/aws-events:requireEventBusPolicySid": true, + "@aws-cdk/core:aspectPrioritiesMutating": true, + "@aws-cdk/aws-dynamodb:retainTableReplica": true, + "@aws-cdk/aws-stepfunctions:useDistributedMapResultWriterV2": true, + "@aws-cdk/s3-notifications:addS3TrustKeyPolicyForSnsSubscriptions": true, + "@aws-cdk/aws-ec2:requirePrivateSubnetsForEgressOnlyInternetGateway": true, + "@aws-cdk/aws-s3:publicAccessBlockedByDefault": true, + "@aws-cdk/aws-lambda:useCdkManagedLogGroup": true + } +} diff --git a/typescript/postgres-lambda/jest.config.js b/typescript/postgres-lambda/jest.config.js new file mode 100644 index 0000000000..08263b8954 --- /dev/null +++ b/typescript/postgres-lambda/jest.config.js @@ -0,0 +1,8 @@ +module.exports = { + testEnvironment: 'node', + roots: ['/test'], + testMatch: ['**/*.test.ts'], + transform: { + '^.+\\.tsx?$': 'ts-jest' + } +}; diff --git a/typescript/postgres-lambda/package.json b/typescript/postgres-lambda/package.json new file mode 100644 index 0000000000..eb09c37f5e --- /dev/null +++ b/typescript/postgres-lambda/package.json @@ -0,0 +1,30 @@ +{ + "name": "postgres-lambda", + "version": "0.1.0", + "bin": { + "postgres-lambda": "bin/postgres-lambda.js" + }, + "scripts": { + "build": "tsc", + "watch": "tsc -w", + "test": "jest", + "cdk": "cdk" + }, + "devDependencies": { + "@types/jest": "^29.5.14", + "@types/node": "22.7.9", + "aws-cdk": "^2.1020.2", + "jest": "^29.7.0", + "ts-jest": "^29.2.5", + "ts-node": "^10.9.2", + "typescript": "~5.6.3" + }, + "dependencies": { + "@aws-cdk/aws-lambda-nodejs": "^1.203.0", + "@types/pg": "^8.15.4", + "aws-cdk-lib": "^2.204.0", + "constructs": "^10.4.2", + "esbuild": "^0.25.6", + "pg": "^8.16.3" + } +} diff --git a/typescript/postgres-lambda/tsconfig.json b/typescript/postgres-lambda/tsconfig.json new file mode 100644 index 0000000000..28bb557fac --- /dev/null +++ b/typescript/postgres-lambda/tsconfig.json @@ -0,0 +1,31 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": [ + "es2022" + ], + "declaration": true, + "strict": true, + "noImplicitAny": true, + "strictNullChecks": true, + "noImplicitThis": true, + "alwaysStrict": true, + "noUnusedLocals": false, + "noUnusedParameters": false, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": false, + "inlineSourceMap": true, + "inlineSources": true, + "experimentalDecorators": true, + "strictPropertyInitialization": false, + "typeRoots": [ + "./node_modules/@types" + ] + }, + "exclude": [ + "node_modules", + "cdk.out" + ] +} From 41c78605af2c842d66684a00ca83819e5ca08404 Mon Sep 17 00:00:00 2001 From: Michael Kaiser Date: Fri, 11 Jul 2025 11:51:50 -0500 Subject: [PATCH 02/23] Add CDK application code: Stack definition and entry point --- .../postgres-lambda/bin/postgres-lambda.ts | 16 ++ .../lib/postgres-lambda-stack.ts | 145 ++++++++++++++++++ .../test/postgres-lambda.test.ts | 17 ++ 3 files changed, 178 insertions(+) create mode 100644 typescript/postgres-lambda/bin/postgres-lambda.ts create mode 100644 typescript/postgres-lambda/lib/postgres-lambda-stack.ts create mode 100644 typescript/postgres-lambda/test/postgres-lambda.test.ts diff --git a/typescript/postgres-lambda/bin/postgres-lambda.ts b/typescript/postgres-lambda/bin/postgres-lambda.ts new file mode 100644 index 0000000000..682eb13ce8 --- /dev/null +++ b/typescript/postgres-lambda/bin/postgres-lambda.ts @@ -0,0 +1,16 @@ +#!/usr/bin/env node +import * as cdk from 'aws-cdk-lib'; +import { PostgresLambdaStack } from '../lib/postgres-lambda-stack'; + +const app = new cdk.App(); +new PostgresLambdaStack(app, 'PostgresLambdaStack', { + /* If you don't specify 'env', this stack will be environment-agnostic. + * Account/Region-dependent features and context lookups will not work, + * but a single synthesized template can be deployed anywhere. */ + + /* Uncomment the next line to specialize this stack for the AWS Account + * and Region that are implied by the current CLI configuration. */ + env: { account: process.env.CDK_DEFAULT_ACCOUNT, region: process.env.CDK_DEFAULT_REGION }, + + /* For more information, see https://docs.aws.amazon.com/cdk/latest/guide/environments.html */ +}); \ No newline at end of file diff --git a/typescript/postgres-lambda/lib/postgres-lambda-stack.ts b/typescript/postgres-lambda/lib/postgres-lambda-stack.ts new file mode 100644 index 0000000000..ef109c4943 --- /dev/null +++ b/typescript/postgres-lambda/lib/postgres-lambda-stack.ts @@ -0,0 +1,145 @@ +import * as cdk from 'aws-cdk-lib'; +import { Construct } from 'constructs'; +import * as ec2 from 'aws-cdk-lib/aws-ec2'; +import * as rds from 'aws-cdk-lib/aws-rds'; +import * as lambda from 'aws-cdk-lib/aws-lambda'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import * as cr from 'aws-cdk-lib/custom-resources'; +import * as path from 'path'; + +export class PostgresLambdaStack extends cdk.Stack { + constructor(scope: Construct, id: string, props?: cdk.StackProps) { + super(scope, id, props); + + // Create a VPC for our application + const vpc = new ec2.Vpc(this, 'PostgresLambdaVpc', { + maxAzs: 2, + natGateways: 1, + }); + + // Create a PostgreSQL Aurora Serverless v2 cluster + const dbCluster = new rds.DatabaseCluster(this, 'PostgresCluster', { + engine: rds.DatabaseClusterEngine.auroraPostgres({ + version: rds.AuroraPostgresEngineVersion.VER_17_4, + }), + vpc: vpc, + writer: rds.ClusterInstance.serverlessV2('writer'), + serverlessV2MinCapacity: 0.5, + serverlessV2MaxCapacity: 1, + defaultDatabaseName: 'demodb', + credentials: rds.Credentials.fromGeneratedSecret('postgres'), + }); + + // Create a Lambda function that calls PostgreSQL + const lambdaToPostgres = new lambda.Function(this, 'LambdaToPostgres', { + runtime: lambda.Runtime.NODEJS_LATEST, + handler: 'index.handler', + code: lambda.Code.fromAsset(path.join(__dirname, '../lambda/lambda-to-postgres')), + vpc, + vpcSubnets: { + subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS, + }, + environment: { + DB_SECRET_ARN: dbCluster.secret?.secretArn || '', + DB_NAME: 'demodb', + }, + timeout: cdk.Duration.seconds(30), + }); + + // Grant Lambda access to the DB + dbCluster.connections.allowDefaultPortTo(lambdaToPostgres); + + // Grant the Lambda function permission to read the database secret + dbCluster.secret?.grantRead(lambdaToPostgres); + + // Create a Lambda function that is called by PostgreSQL + const postgresFunction = new lambda.Function(this, 'PostgresFunction', { + runtime: lambda.Runtime.NODEJS_LATEST, + handler: 'index.handler', + code: lambda.Code.fromAsset(path.join(__dirname, '../lambda/postgres-to-lambda')), + environment: { + FUNCTION_NAME: 'PostgresFunction', + }, + timeout: cdk.Duration.seconds(30), + }); + + + // Create a role for PostgreSQL to assume to invoke Lambda + const postgresLambdaRole = new iam.Role(this, 'PostgresLambdaRole', { + assumedBy: new iam.ServicePrincipal('rds.amazonaws.com'), + }); + + postgresFunction.grantInvoke(postgresLambdaRole); + + + const l1DbCluster = dbCluster.node.defaultChild as rds.CfnDBCluster + const exisitingProperty = (l1DbCluster.associatedRoles as []) || []; + console.log(exisitingProperty); + + const newRole: { FeatureName: string, RoleArn: string } = { + FeatureName: "Lambda", // Changed to PascalCase + RoleArn: postgresLambdaRole.roleArn // Changed to PascalCase + }; + + const updatedRoles: { [key in 'featureName' | 'FeatureName' | 'roleArn' | 'RoleArn']?: string; }[] = [...exisitingProperty, newRole]; + + l1DbCluster.addPropertyOverride('AssociatedRoles', updatedRoles); + + // Create Lambda function for PostgreSQL setup + const setupFunction = new lambda.Function(this, 'PostgresSetupFunction', { + runtime: lambda.Runtime.NODEJS_LATEST, + handler: 'index.handler', + code: lambda.Code.fromAsset(path.join(__dirname, '../lambda/postgres-setup')), + vpc, + vpcSubnets: { + subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS, + }, + environment: { + DB_SECRET_ARN: dbCluster.secret?.secretArn || '', + DB_NAME: 'demodb', + POSTGRES_FUNCTION_NAME: postgresFunction.functionName, + AWS_REGION: this.region, + }, + timeout: cdk.Duration.minutes(5), + }); + + // Grant setup function access to the DB and secrets + dbCluster.connections.allowDefaultPortTo(setupFunction); + dbCluster.secret?.grantRead(setupFunction); + + // Create custom resource to trigger setup + const setupProvider = new cr.Provider(this, 'PostgresSetupProvider', { + onEventHandler: setupFunction, + }); + + new cdk.CustomResource(this, 'PostgresSetupResource', { + serviceToken: setupProvider.serviceToken, + }); + + // Output the database endpoint and secret ARN + new cdk.CfnOutput(this, 'DBClusterEndpoint', { + value: dbCluster.clusterEndpoint.hostname, + description: 'The endpoint of the database cluster', + }); + + new cdk.CfnOutput(this, 'DBSecretArn', { + value: dbCluster.secret?.secretArn || 'No secret created', + description: 'The ARN of the database credentials secret', + }); + + new cdk.CfnOutput(this, 'LambdaToPostgresFunctionName', { + value: lambdaToPostgres.functionName, + description: 'The name of the Lambda function that calls PostgreSQL', + }); + + new cdk.CfnOutput(this, 'PostgresFunctionName', { + value: postgresFunction.functionName, + description: 'The name of the Lambda function that is called by PostgreSQL', + }); + + new cdk.CfnOutput(this, 'PostgresLambdaRoleArn', { + value: postgresLambdaRole.roleArn, + description: 'The ARN of the role that PostgreSQL can assume to invoke Lambda', + }); + } +} \ No newline at end of file diff --git a/typescript/postgres-lambda/test/postgres-lambda.test.ts b/typescript/postgres-lambda/test/postgres-lambda.test.ts new file mode 100644 index 0000000000..4c261f22e3 --- /dev/null +++ b/typescript/postgres-lambda/test/postgres-lambda.test.ts @@ -0,0 +1,17 @@ +// import * as cdk from 'aws-cdk-lib'; +// import { Template } from 'aws-cdk-lib/assertions'; +// import * as PostgresLambda from '../lib/postgres-lambda-stack'; + +// example test. To run these tests, uncomment this file along with the +// example resource in lib/postgres-lambda-stack.ts +test('SQS Queue Created', () => { +// const app = new cdk.App(); +// // WHEN +// const stack = new PostgresLambda.PostgresLambdaStack(app, 'MyTestStack'); +// // THEN +// const template = Template.fromStack(stack); + +// template.hasResourceProperties('AWS::SQS::Queue', { +// VisibilityTimeout: 300 +// }); +}); From 55fdf5289b41536bafeb426728a6e8029c61c86e Mon Sep 17 00:00:00 2001 From: Michael Kaiser Date: Fri, 11 Jul 2025 11:52:00 -0500 Subject: [PATCH 03/23] Add Lambda function package configurations --- .../lambda/lambda-to-postgres/package.json | 10 ++++++++++ .../postgres-lambda/lambda/postgres-setup/package.json | 9 +++++++++ .../lambda/postgres-to-lambda/package.json | 7 +++++++ 3 files changed, 26 insertions(+) create mode 100644 typescript/postgres-lambda/lambda/lambda-to-postgres/package.json create mode 100644 typescript/postgres-lambda/lambda/postgres-setup/package.json create mode 100644 typescript/postgres-lambda/lambda/postgres-to-lambda/package.json diff --git a/typescript/postgres-lambda/lambda/lambda-to-postgres/package.json b/typescript/postgres-lambda/lambda/lambda-to-postgres/package.json new file mode 100644 index 0000000000..360d335a3b --- /dev/null +++ b/typescript/postgres-lambda/lambda/lambda-to-postgres/package.json @@ -0,0 +1,10 @@ +{ + "name": "lambda-to-postgres", + "version": "1.0.0", + "description": "Lambda function that calls PostgreSQL", + "main": "index.js", + "dependencies": { + "pg": "^8.11.3", + "@aws-sdk/client-secrets-manager": "^3.450.0" + } +} diff --git a/typescript/postgres-lambda/lambda/postgres-setup/package.json b/typescript/postgres-lambda/lambda/postgres-setup/package.json new file mode 100644 index 0000000000..719dc2b692 --- /dev/null +++ b/typescript/postgres-lambda/lambda/postgres-setup/package.json @@ -0,0 +1,9 @@ +{ + "name": "postgres-setup", + "version": "1.0.0", + "dependencies": { + "pg": "^8.11.3", + "@aws-sdk/client-secrets-manager": "^3.0.0", + "node-fetch": "^3.3.2" + } +} \ No newline at end of file diff --git a/typescript/postgres-lambda/lambda/postgres-to-lambda/package.json b/typescript/postgres-lambda/lambda/postgres-to-lambda/package.json new file mode 100644 index 0000000000..a061a15f5f --- /dev/null +++ b/typescript/postgres-lambda/lambda/postgres-to-lambda/package.json @@ -0,0 +1,7 @@ +{ + "name": "postgres-to-lambda", + "version": "1.0.0", + "description": "Lambda function that is called by PostgreSQL", + "main": "index.js", + "dependencies": {} +} From ecf4cb760e3ad9a0b36fdff539ef27b4fef513aa Mon Sep 17 00:00:00 2001 From: Michael Kaiser Date: Fri, 11 Jul 2025 11:52:18 -0500 Subject: [PATCH 04/23] Add documentation: README and SUMMARY --- typescript/postgres-lambda/README.md | 268 ++++++++++++++++++++++++++ typescript/postgres-lambda/SUMMARY.md | 50 +++++ 2 files changed, 318 insertions(+) create mode 100644 typescript/postgres-lambda/README.md create mode 100644 typescript/postgres-lambda/SUMMARY.md diff --git a/typescript/postgres-lambda/README.md b/typescript/postgres-lambda/README.md new file mode 100644 index 0000000000..105e9b079f --- /dev/null +++ b/typescript/postgres-lambda/README.md @@ -0,0 +1,268 @@ +# PostgreSQL and Lambda Integration Example + +A complete AWS CDK example demonstrating bidirectional integration between Aurora PostgreSQL Serverless v2 and AWS Lambda functions. + +## What This Example Demonstrates + +- **Lambda → PostgreSQL**: Lambda function that connects to and queries PostgreSQL +- **PostgreSQL → Lambda**: PostgreSQL database that invokes Lambda functions using the `aws_lambda` extension +- **Secure Architecture**: Private subnets, IAM roles, and Secrets Manager integration +- **Production-Ready**: Includes error handling, connection pooling, and security best practices +- **Automated Setup**: Custom CDK resource automatically configures PostgreSQL extensions and functions + +## Architecture + +```mermaid +graph TD + subgraph VPC + subgraph "Private Subnet" + DB[Aurora PostgreSQL\nServerless v2] + L1[Lambda Function\nLambdaToPostgres] + L2[Lambda Function\nPostgresFunction] + L3[Lambda Function\nPostgresSetup] + end + end + + L1 -->|"(1) Connect and Query"| DB + DB -->|"(2) Invoke via aws_lambda extension"| L2 + L2 -->|"(3) Return Result"| DB + L3 -->|"(4) Setup Extensions & Functions"| DB + + SM[AWS Secrets Manager] -->|Provide Credentials| L1 + SM -->|Provide Credentials| L3 + + style DB fill:#E3F2FD,stroke:#1976D2,stroke-width:2px,color:#000 + style L1 fill:#FFF3E0,stroke:#F57C00,stroke-width:2px,color:#000 + style L2 fill:#FFF3E0,stroke:#F57C00,stroke-width:2px,color:#000 + style L3 fill:#E8F5E8,stroke:#4CAF50,stroke-width:2px,color:#000 + style SM fill:#F3E5F5,stroke:#7B1FA2,stroke-width:2px,color:#000 +``` + +**Components:** +- Aurora PostgreSQL Serverless v2 cluster (private subnet) +- Lambda function for database operations (`LambdaToPostgres`) +- Lambda function invokable from PostgreSQL (`PostgresFunction`) +- Lambda function for automated setup (`PostgresSetupFunction`) +- IAM roles with least-privilege permissions +- Security groups for network access control +- AWS Secrets Manager for credential storage +- Custom CDK resource for automated PostgreSQL configuration + +## Quick Start + +### Prerequisites + +- AWS CDK v2 installed (`npm install -g aws-cdk`) +- Node.js 18.x or later +- AWS CLI configured with appropriate credentials + +### Deploy + +```bash +# Install dependencies +npm install + +# Deploy the stack (setup is now automated!) +npx cdk deploy +``` + +The deployment will automatically: +- Create the Aurora PostgreSQL cluster +- Deploy all Lambda functions +- Configure PostgreSQL extensions and functions +- Set up all necessary permissions + +No manual setup required! 🎉 + +## Testing + +### Test Lambda → PostgreSQL + +Using the provided test script: +```bash +./test-lambda.sh --function-name --message "Hello World" +``` + +Or using AWS CLI directly: +```bash +aws lambda invoke \ + --function-name \ + --payload '{"message": "Hello from CLI!"}' \ + response.json && cat response.json +``` + +### Test PostgreSQL → Lambda + +Connect to PostgreSQL and test the functions: +```bash +psql -h -U postgres -d demodb +``` + +```sql +-- Test the PostgreSQL to Lambda integration +SELECT process_data('{"id": 123, "value": "test"}'::JSONB); +SELECT transform_data('{"id": 456, "value": "hello world"}'::JSONB); +SELECT validate_data('{"id": 789, "value": "valid data"}'::JSONB); +``` + +## How It Works + +### Automated Setup Process + +1. **CDK Deployment**: Stack creates all resources including a setup Lambda function +2. **Custom Resource**: Triggers the setup Lambda after database is ready +3. **Extension Installation**: Setup function creates the `aws_lambda` extension +4. **Function Creation**: Creates SQL functions that wrap Lambda invocations +5. **Ready to Use**: Database is immediately ready for bidirectional Lambda integration + +### Lambda to PostgreSQL Flow + +1. **Credential Retrieval**: Function retrieves DB credentials from Secrets Manager +2. **Connection**: Establishes secure SSL connection to PostgreSQL +3. **Table Management**: Creates demo table if it doesn't exist +4. **Data Operations**: Inserts message and queries recent records +5. **Response**: Returns formatted results with error handling + +### PostgreSQL to Lambda Flow + +1. **Extension Setup**: Uses `aws_lambda` extension for Lambda invocation (automated) +2. **Function Creation**: SQL functions wrap Lambda calls with proper ARN construction (automated) +3. **Event Processing**: Lambda receives structured JSON events from PostgreSQL +4. **Result Return**: Lambda response becomes available in SQL query results + +## Project Structure + +``` +├── bin/ # CDK app entry point +├── lib/ # CDK stack definition +├── lambda/ # Lambda function source code +│ ├── lambda-to-postgres/ # Lambda that calls PostgreSQL +│ ├── postgres-to-lambda/ # Lambda called by PostgreSQL +│ └── postgres-setup/ # Lambda for automated setup +├── test/ # Unit tests +├── setup-postgres-lambda.sql # Reference SQL (now automated) +├── test-lambda.sh # Lambda testing script +└── README.md # This file +``` + +## Configuration + +### Environment Variables + +The Lambda functions use these environment variables (set automatically by CDK): + +- `DB_SECRET_ARN`: ARN of the database credentials secret +- `DB_NAME`: Database name (default: `demodb`) +- `POSTGRES_FUNCTION_NAME`: Name of the Lambda function called by PostgreSQL +- `AWS_REGION`: AWS region for Lambda ARN construction + +### Customization + +- **Database Configuration**: Modify `lib/postgres-lambda-stack.ts` +- **Lambda Logic**: Update files in `lambda/` directories +- **Setup SQL**: Customize `lambda/postgres-setup/index.js` + +## Security Features + +✅ **Network Security** +- Database in private subnets +- Security groups with minimal required access +- No direct internet access to database + +✅ **Access Control** +- IAM roles with least-privilege permissions +- Secrets Manager for credential storage +- SSL/TLS encryption for database connections + +✅ **Monitoring** +- CloudWatch logs for all Lambda functions +- Database performance insights available +- VPC Flow Logs (can be enabled) + +## Production Considerations + +Before using in production: + +- [ ] Enable SSL certificate validation (`rejectUnauthorized: true`) +- [ ] Implement connection pooling (consider RDS Proxy) +- [ ] Set up proper monitoring and alerting +- [ ] Configure backup and disaster recovery +- [ ] Review and tighten IAM policies +- [ ] Enable database encryption at rest +- [ ] Set up VPC endpoints for AWS services +- [ ] Implement proper error handling and retry logic + +## Troubleshooting + +### Common Issues + +**Connection Timeouts** +- Check security group rules +- Verify Lambda is in correct VPC/subnets +- Confirm database is running + +**Permission Errors** +- Verify IAM roles have required permissions +- Check Secrets Manager access +- Confirm Lambda execution role + +**Setup Function Issues** +- Check CloudWatch logs for the PostgresSetupFunction +- Verify custom resource completed successfully +- Ensure database is accessible from setup Lambda + +### Useful Commands + +```bash +# Build and watch for changes +npm run watch + +# Run tests +npm run test + +# View CloudFormation template +npx cdk synth + +# Compare deployed vs current state +npx cdk diff + +# View stack outputs +aws cloudformation describe-stacks --stack-name PostgresLambdaStack --query 'Stacks[0].Outputs' + +# Check setup function logs +aws logs describe-log-groups --log-group-name-prefix /aws/lambda/PostgresLambdaStack-PostgresSetupFunction +``` + +## Cleanup + +```bash +npx cdk destroy +``` + +**Note**: This will delete all resources including the database and any data stored in it. + +## Cost Optimization + +- Aurora Serverless v2 scales to zero when not in use +- Lambda functions only charge for execution time +- Setup function runs only once during deployment +- Consider Reserved Capacity for consistent workloads +- Monitor usage with AWS Cost Explorer + +## Related Examples + +- [Lambda with RDS Proxy](../lambda-rds-proxy/) +- [Aurora Serverless v1](../aurora-serverless-v1/) +- [PostgreSQL with CDK](../postgresql-cdk/) + +## Contributing + +1. Fork the repository +2. Create a feature branch +3. Make your changes +4. Add tests if applicable +5. Submit a pull request + +## License + +This example is provided under the MIT-0 License. See the LICENSE file for details. \ No newline at end of file diff --git a/typescript/postgres-lambda/SUMMARY.md b/typescript/postgres-lambda/SUMMARY.md new file mode 100644 index 0000000000..aab23f5e01 --- /dev/null +++ b/typescript/postgres-lambda/SUMMARY.md @@ -0,0 +1,50 @@ +# PostgreSQL and Lambda Integration Example - Summary + +This CDK example demonstrates the integration between AWS Lambda and Aurora PostgreSQL Serverless v2. It showcases two key integration patterns: + +## 1. Lambda to PostgreSQL + +The first pattern demonstrates how a Lambda function can connect to and interact with a PostgreSQL database: + +- The Lambda function (`LambdaToPostgres`) retrieves database credentials from AWS Secrets Manager +- It establishes a connection to the PostgreSQL database +- It creates a table if it doesn't exist, inserts data, and queries the database +- The function returns the query results + +## 2. PostgreSQL to Lambda + +The second pattern demonstrates how PostgreSQL can invoke a Lambda function: + +- PostgreSQL uses the `aws_lambda` extension to call Lambda functions +- The Lambda function (`PostgresFunction`) receives data from PostgreSQL +- It processes the data based on the action specified in the event +- It returns results that can be used in SQL queries + +## Security Features + +The example implements several security best practices: + +- The database is deployed in a private subnet +- Security groups restrict access to the database +- Credentials are stored in AWS Secrets Manager +- IAM roles limit permissions to only what's necessary + +## Helper Scripts + +The example includes several helper scripts: + +- `test-lambda.sh`: For testing the Lambda functions +- `connect-to-postgres.sh`: For connecting to the PostgreSQL database +- `setup-postgres-lambda.sql`: For setting up the PostgreSQL database to call Lambda + +## Deployment + +The example can be deployed with standard CDK commands: + +```bash +npm install +npm run build +npx cdk deploy +``` + +After deployment, users need to set up the PostgreSQL database to call Lambda by creating the `aws_lambda` extension and defining functions that invoke Lambda. From 75cc057597f0ac3d28e55cab5036c2f229ad979c Mon Sep 17 00:00:00 2001 From: Michael Kaiser Date: Fri, 11 Jul 2025 11:52:37 -0500 Subject: [PATCH 05/23] Add utility scripts and SQL setup file --- .../postgres-lambda/setup-postgres-lambda.sql | 41 ++++++++++ typescript/postgres-lambda/test-lambda.sh | 75 +++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 typescript/postgres-lambda/setup-postgres-lambda.sql create mode 100755 typescript/postgres-lambda/test-lambda.sh diff --git a/typescript/postgres-lambda/setup-postgres-lambda.sql b/typescript/postgres-lambda/setup-postgres-lambda.sql new file mode 100644 index 0000000000..97ecd8b36d --- /dev/null +++ b/typescript/postgres-lambda/setup-postgres-lambda.sql @@ -0,0 +1,41 @@ +-- SQL script to set up PostgreSQL to call Lambda +-- Replace with your AWS region (e.g., us-east-1) +-- Replace with the name of your Lambda function + +-- Create the aws_lambda extension +CREATE EXTENSION IF NOT EXISTS aws_lambda CASCADE; + +-- Create a function to process data using Lambda +CREATE OR REPLACE FUNCTION process_data(data JSONB) +RETURNS JSONB AS $$ +SELECT payload FROM aws_lambda.invoke( + aws_commons.create_lambda_function_arn('', ''), + json_build_object('action', 'process', 'data', data)::text, + 'Event' +); +$$ LANGUAGE SQL; + +-- Create a function to transform data using Lambda +CREATE OR REPLACE FUNCTION transform_data(data JSONB) +RETURNS JSONB AS $$ +SELECT payload FROM aws_lambda.invoke( + aws_commons.create_lambda_function_arn('', ''), + json_build_object('action', 'transform', 'data', data)::text, + 'Event' +); +$$ LANGUAGE SQL; + +-- Create a function to validate data using Lambda +CREATE OR REPLACE FUNCTION validate_data(data JSONB) +RETURNS JSONB AS $$ +SELECT payload FROM aws_lambda.invoke( + aws_commons.create_lambda_function_arn('', ''), + json_build_object('action', 'validate', 'data', data)::text, + 'Event' +); +$$ LANGUAGE SQL; + +-- Test the functions +SELECT process_data('{"id": 123, "value": "test"}'::JSONB); +SELECT transform_data('{"id": 456, "value": "hello world"}'::JSONB); +SELECT validate_data('{"id": 789, "value": "valid data"}'::JSONB); diff --git a/typescript/postgres-lambda/test-lambda.sh b/typescript/postgres-lambda/test-lambda.sh new file mode 100755 index 0000000000..637b859ae4 --- /dev/null +++ b/typescript/postgres-lambda/test-lambda.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +# Test script for PostgreSQL Lambda integration + +# Check if jq is installed +if ! command -v jq &> /dev/null; then + echo "Error: jq is required but not installed. Please install jq first." + exit 1 +fi + +# Function to display usage +function display_usage { + echo "Usage: $0 [options]" + echo "Options:" + echo " -f, --function-name NAME Lambda function name to test" + echo " -m, --message MESSAGE Message to send to Lambda (default: 'Test message')" + echo " -h, --help Display this help message" + exit 1 +} + +# Default values +MESSAGE="Test message" + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + key="$1" + case $key in + -f|--function-name) + FUNCTION_NAME="$2" + shift + shift + ;; + -m|--message) + MESSAGE="$2" + shift + shift + ;; + -h|--help) + display_usage + ;; + *) + echo "Unknown option: $1" + display_usage + ;; + esac +done + +# Check if function name is provided +if [ -z "$FUNCTION_NAME" ]; then + echo "Error: Lambda function name is required." + display_usage +fi + +# Create payload +PAYLOAD=$(echo "{\"message\": \"$MESSAGE\"}" | jq -c .) + +# Create temporary file for response +RESPONSE_FILE=$(mktemp) + +echo "Invoking Lambda function: $FUNCTION_NAME" +echo "Payload: $PAYLOAD" + +# Invoke Lambda function +aws lambda invoke \ + --function-name "$FUNCTION_NAME" \ + --payload "$PAYLOAD" \ + --cli-binary-format raw-in-base64-out \ + "$RESPONSE_FILE" + +# Display response +echo "Response:" +cat "$RESPONSE_FILE" | jq . + +# Clean up +rm "$RESPONSE_FILE" From 929750adffbb8a4508eaef92672bf595084fbf46 Mon Sep 17 00:00:00 2001 From: Michael Kaiser Date: Fri, 11 Jul 2025 12:36:11 -0500 Subject: [PATCH 06/23] Implement yarn workspaces for Lambda functions management and update .gitignore --- typescript/postgres-lambda/.gitignore | 27 ++++++++++ typescript/postgres-lambda/README.md | 52 ++++++++++++++----- .../lambda/lambda-to-postgres/package.json | 12 +++-- .../lambda/postgres-setup/package.json | 15 ++++-- .../lambda/postgres-to-lambda/package.json | 12 +++-- .../lib/postgres-lambda-stack.ts | 3 +- typescript/postgres-lambda/package.json | 12 +++-- 7 files changed, 102 insertions(+), 31 deletions(-) diff --git a/typescript/postgres-lambda/.gitignore b/typescript/postgres-lambda/.gitignore index f60797b6a9..aacf8b71fe 100644 --- a/typescript/postgres-lambda/.gitignore +++ b/typescript/postgres-lambda/.gitignore @@ -2,7 +2,34 @@ !jest.config.js *.d.ts node_modules +**/node_modules # CDK asset staging directory .cdk.staging cdk.out + +# Yarn specific +.yarn/* +!.yarn/patches +!.yarn/plugins +!.yarn/releases +!.yarn/sdks +!.yarn/versions +.pnp.* +yarn-debug.log* +yarn-error.log* + +# Build artifacts +dist/ +build/ +*.tsbuildinfo + +# IDE files +.idea/ +.vscode/ +*.swp +*.swo + +# OS files +.DS_Store +Thumbs.db diff --git a/typescript/postgres-lambda/README.md b/typescript/postgres-lambda/README.md index 105e9b079f..a5ca0640d0 100644 --- a/typescript/postgres-lambda/README.md +++ b/typescript/postgres-lambda/README.md @@ -9,6 +9,7 @@ A complete AWS CDK example demonstrating bidirectional integration between Auror - **Secure Architecture**: Private subnets, IAM roles, and Secrets Manager integration - **Production-Ready**: Includes error handling, connection pooling, and security best practices - **Automated Setup**: Custom CDK resource automatically configures PostgreSQL extensions and functions +- **Yarn Workspaces**: Organized monorepo structure for managing multiple Lambda functions ## Architecture @@ -23,10 +24,10 @@ graph TD end end - L1 -->|"(1) Connect and Query"| DB - DB -->|"(2) Invoke via aws_lambda extension"| L2 - L2 -->|"(3) Return Result"| DB - L3 -->|"(4) Setup Extensions & Functions"| DB + L1 -->|"1. Connect and Query"| DB + DB -->|"2. Invoke via aws_lambda extension"| L2 + L2 -->|"3. Return Result"| DB + L3 -->|"4. Setup Extensions & Functions"| DB SM[AWS Secrets Manager] -->|Provide Credentials| L1 SM -->|Provide Credentials| L3 @@ -54,16 +55,17 @@ graph TD - AWS CDK v2 installed (`npm install -g aws-cdk`) - Node.js 18.x or later +- Yarn package manager installed - AWS CLI configured with appropriate credentials ### Deploy ```bash -# Install dependencies -npm install +# Install dependencies using yarn workspaces +yarn install # Deploy the stack (setup is now automated!) -npx cdk deploy +yarn cdk deploy ``` The deployment will automatically: @@ -145,6 +147,30 @@ SELECT validate_data('{"id": 789, "value": "valid data"}'::JSONB); └── README.md # This file ``` +## Yarn Workspaces + +This project uses Yarn Workspaces to manage multiple packages in a monorepo structure: + +```bash +# List all workspaces +yarn workspaces list + +# Run a command in all Lambda workspaces +yarn workspaces foreach -v --include '@lambda/*' run + +# Run a command in a specific workspace +yarn workspace @lambda/lambda-to-postgres run + +# Install dependencies for all workspaces +yarn install +``` + +The workspace structure allows for: +- Shared dependencies between packages +- Individual package management +- Simplified build and deployment process +- Better organization of Lambda functions + ## Configuration ### Environment Variables @@ -215,16 +241,16 @@ Before using in production: ```bash # Build and watch for changes -npm run watch +yarn watch # Run tests -npm run test +yarn test # View CloudFormation template -npx cdk synth +yarn cdk synth # Compare deployed vs current state -npx cdk diff +yarn cdk diff # View stack outputs aws cloudformation describe-stacks --stack-name PostgresLambdaStack --query 'Stacks[0].Outputs' @@ -236,7 +262,7 @@ aws logs describe-log-groups --log-group-name-prefix /aws/lambda/PostgresLambdaS ## Cleanup ```bash -npx cdk destroy +yarn cdk destroy ``` **Note**: This will delete all resources including the database and any data stored in it. @@ -265,4 +291,4 @@ npx cdk destroy ## License -This example is provided under the MIT-0 License. See the LICENSE file for details. \ No newline at end of file +This example is provided under the MIT-0 License. See the LICENSE file for details. diff --git a/typescript/postgres-lambda/lambda/lambda-to-postgres/package.json b/typescript/postgres-lambda/lambda/lambda-to-postgres/package.json index 360d335a3b..80a154e90b 100644 --- a/typescript/postgres-lambda/lambda/lambda-to-postgres/package.json +++ b/typescript/postgres-lambda/lambda/lambda-to-postgres/package.json @@ -1,10 +1,14 @@ { - "name": "lambda-to-postgres", + "name": "@lambda/lambda-to-postgres", "version": "1.0.0", - "description": "Lambda function that calls PostgreSQL", + "description": "Lambda function that connects to PostgreSQL", "main": "index.js", + "scripts": { + "build": "echo 'No build needed for JavaScript files'", + "test": "echo \"Error: no test specified\" && exit 1" + }, "dependencies": { - "pg": "^8.11.3", - "@aws-sdk/client-secrets-manager": "^3.450.0" + "pg": "^8.16.3", + "aws-sdk": "^2.1377.0" } } diff --git a/typescript/postgres-lambda/lambda/postgres-setup/package.json b/typescript/postgres-lambda/lambda/postgres-setup/package.json index 719dc2b692..27b675102b 100644 --- a/typescript/postgres-lambda/lambda/postgres-setup/package.json +++ b/typescript/postgres-lambda/lambda/postgres-setup/package.json @@ -1,9 +1,14 @@ { - "name": "postgres-setup", + "name": "@lambda/postgres-setup", "version": "1.0.0", + "description": "Lambda function for automated PostgreSQL setup", + "main": "index.js", + "scripts": { + "build": "echo 'No build needed for JavaScript files'", + "test": "echo \"Error: no test specified\" && exit 1" + }, "dependencies": { - "pg": "^8.11.3", - "@aws-sdk/client-secrets-manager": "^3.0.0", - "node-fetch": "^3.3.2" + "pg": "^8.16.3", + "aws-sdk": "^2.1377.0" } -} \ No newline at end of file +} diff --git a/typescript/postgres-lambda/lambda/postgres-to-lambda/package.json b/typescript/postgres-lambda/lambda/postgres-to-lambda/package.json index a061a15f5f..62eebc805f 100644 --- a/typescript/postgres-lambda/lambda/postgres-to-lambda/package.json +++ b/typescript/postgres-lambda/lambda/postgres-to-lambda/package.json @@ -1,7 +1,13 @@ { - "name": "postgres-to-lambda", + "name": "@lambda/postgres-to-lambda", "version": "1.0.0", - "description": "Lambda function that is called by PostgreSQL", + "description": "Lambda function called by PostgreSQL", "main": "index.js", - "dependencies": {} + "scripts": { + "build": "echo 'No build needed for JavaScript files'", + "test": "echo \"Error: no test specified\" && exit 1" + }, + "dependencies": { + "aws-sdk": "^2.1377.0" + } } diff --git a/typescript/postgres-lambda/lib/postgres-lambda-stack.ts b/typescript/postgres-lambda/lib/postgres-lambda-stack.ts index ef109c4943..def886744e 100644 --- a/typescript/postgres-lambda/lib/postgres-lambda-stack.ts +++ b/typescript/postgres-lambda/lib/postgres-lambda-stack.ts @@ -98,7 +98,6 @@ export class PostgresLambdaStack extends cdk.Stack { DB_SECRET_ARN: dbCluster.secret?.secretArn || '', DB_NAME: 'demodb', POSTGRES_FUNCTION_NAME: postgresFunction.functionName, - AWS_REGION: this.region, }, timeout: cdk.Duration.minutes(5), }); @@ -142,4 +141,4 @@ export class PostgresLambdaStack extends cdk.Stack { description: 'The ARN of the role that PostgreSQL can assume to invoke Lambda', }); } -} \ No newline at end of file +} diff --git a/typescript/postgres-lambda/package.json b/typescript/postgres-lambda/package.json index eb09c37f5e..a6a303b0f6 100644 --- a/typescript/postgres-lambda/package.json +++ b/typescript/postgres-lambda/package.json @@ -1,11 +1,17 @@ { "name": "postgres-lambda", "version": "0.1.0", + "private": true, + "workspaces": [ + ".", + "lambda/*" + ], "bin": { "postgres-lambda": "bin/postgres-lambda.js" }, "scripts": { - "build": "tsc", + "build": "tsc && yarn workspaces foreach -v -A run build", + "build:lambda": "yarn workspaces foreach -v -A run build", "watch": "tsc -w", "test": "jest", "cdk": "cdk" @@ -21,10 +27,8 @@ }, "dependencies": { "@aws-cdk/aws-lambda-nodejs": "^1.203.0", - "@types/pg": "^8.15.4", "aws-cdk-lib": "^2.204.0", "constructs": "^10.4.2", - "esbuild": "^0.25.6", - "pg": "^8.16.3" + "esbuild": "^0.25.6" } } From ce066a69a1693f827709d8f6195fa9295f7aa19d Mon Sep 17 00:00:00 2001 From: Michael Kaiser Date: Fri, 11 Jul 2025 12:39:48 -0500 Subject: [PATCH 07/23] Fix Lambda bundling by using NodejsFunction --- .../lib/postgres-lambda-stack.ts | 39 ++++++++++++------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/typescript/postgres-lambda/lib/postgres-lambda-stack.ts b/typescript/postgres-lambda/lib/postgres-lambda-stack.ts index def886744e..8a8f8a824b 100644 --- a/typescript/postgres-lambda/lib/postgres-lambda-stack.ts +++ b/typescript/postgres-lambda/lib/postgres-lambda-stack.ts @@ -6,6 +6,7 @@ import * as lambda from 'aws-cdk-lib/aws-lambda'; import * as iam from 'aws-cdk-lib/aws-iam'; import * as cr from 'aws-cdk-lib/custom-resources'; import * as path from 'path'; +import * as nodejs from 'aws-cdk-lib/aws-lambda-nodejs'; export class PostgresLambdaStack extends cdk.Stack { constructor(scope: Construct, id: string, props?: cdk.StackProps) { @@ -30,11 +31,10 @@ export class PostgresLambdaStack extends cdk.Stack { credentials: rds.Credentials.fromGeneratedSecret('postgres'), }); - // Create a Lambda function that calls PostgreSQL - const lambdaToPostgres = new lambda.Function(this, 'LambdaToPostgres', { + // Create a Lambda function that calls PostgreSQL using NodejsFunction + const lambdaToPostgres = new nodejs.NodejsFunction(this, 'LambdaToPostgres', { runtime: lambda.Runtime.NODEJS_LATEST, - handler: 'index.handler', - code: lambda.Code.fromAsset(path.join(__dirname, '../lambda/lambda-to-postgres')), + entry: path.join(__dirname, '../lambda/lambda-to-postgres/index.js'), vpc, vpcSubnets: { subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS, @@ -43,6 +43,11 @@ export class PostgresLambdaStack extends cdk.Stack { DB_SECRET_ARN: dbCluster.secret?.secretArn || '', DB_NAME: 'demodb', }, + bundling: { + externalModules: [ + 'aws-sdk', // Use the AWS SDK available in the Lambda runtime + ], + }, timeout: cdk.Duration.seconds(30), }); @@ -52,18 +57,21 @@ export class PostgresLambdaStack extends cdk.Stack { // Grant the Lambda function permission to read the database secret dbCluster.secret?.grantRead(lambdaToPostgres); - // Create a Lambda function that is called by PostgreSQL - const postgresFunction = new lambda.Function(this, 'PostgresFunction', { + // Create a Lambda function that is called by PostgreSQL using NodejsFunction + const postgresFunction = new nodejs.NodejsFunction(this, 'PostgresFunction', { runtime: lambda.Runtime.NODEJS_LATEST, - handler: 'index.handler', - code: lambda.Code.fromAsset(path.join(__dirname, '../lambda/postgres-to-lambda')), + entry: path.join(__dirname, '../lambda/postgres-to-lambda/index.js'), environment: { FUNCTION_NAME: 'PostgresFunction', }, + bundling: { + externalModules: [ + 'aws-sdk', // Use the AWS SDK available in the Lambda runtime + ], + }, timeout: cdk.Duration.seconds(30), }); - // Create a role for PostgreSQL to assume to invoke Lambda const postgresLambdaRole = new iam.Role(this, 'PostgresLambdaRole', { assumedBy: new iam.ServicePrincipal('rds.amazonaws.com'), @@ -71,7 +79,6 @@ export class PostgresLambdaStack extends cdk.Stack { postgresFunction.grantInvoke(postgresLambdaRole); - const l1DbCluster = dbCluster.node.defaultChild as rds.CfnDBCluster const exisitingProperty = (l1DbCluster.associatedRoles as []) || []; console.log(exisitingProperty); @@ -85,11 +92,10 @@ export class PostgresLambdaStack extends cdk.Stack { l1DbCluster.addPropertyOverride('AssociatedRoles', updatedRoles); - // Create Lambda function for PostgreSQL setup - const setupFunction = new lambda.Function(this, 'PostgresSetupFunction', { + // Create Lambda function for PostgreSQL setup using NodejsFunction + const setupFunction = new nodejs.NodejsFunction(this, 'PostgresSetupFunction', { runtime: lambda.Runtime.NODEJS_LATEST, - handler: 'index.handler', - code: lambda.Code.fromAsset(path.join(__dirname, '../lambda/postgres-setup')), + entry: path.join(__dirname, '../lambda/postgres-setup/index.js'), vpc, vpcSubnets: { subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS, @@ -99,6 +105,11 @@ export class PostgresLambdaStack extends cdk.Stack { DB_NAME: 'demodb', POSTGRES_FUNCTION_NAME: postgresFunction.functionName, }, + bundling: { + externalModules: [ + 'aws-sdk', // Use the AWS SDK available in the Lambda runtime + ], + }, timeout: cdk.Duration.minutes(5), }); From 522ce5a8b5ec42024918792e4ec310ccfe6adc91 Mon Sep 17 00:00:00 2001 From: Michael Kaiser Date: Fri, 11 Jul 2025 12:46:49 -0500 Subject: [PATCH 08/23] Fix Lambda bundling by using Docker to install pg module --- typescript/postgres-lambda/.gitignore | 4 +- .../lib/postgres-lambda-stack.ts | 65 ++++++++++++------- 2 files changed, 43 insertions(+), 26 deletions(-) diff --git a/typescript/postgres-lambda/.gitignore b/typescript/postgres-lambda/.gitignore index aacf8b71fe..abb73b4410 100644 --- a/typescript/postgres-lambda/.gitignore +++ b/typescript/postgres-lambda/.gitignore @@ -2,7 +2,9 @@ !jest.config.js *.d.ts node_modules -**/node_modules +# Allow node_modules in Lambda directories +!lambda/*/node_modules +!lambda/*/*.js # CDK asset staging directory .cdk.staging diff --git a/typescript/postgres-lambda/lib/postgres-lambda-stack.ts b/typescript/postgres-lambda/lib/postgres-lambda-stack.ts index 8a8f8a824b..61b38f9782 100644 --- a/typescript/postgres-lambda/lib/postgres-lambda-stack.ts +++ b/typescript/postgres-lambda/lib/postgres-lambda-stack.ts @@ -6,7 +6,6 @@ import * as lambda from 'aws-cdk-lib/aws-lambda'; import * as iam from 'aws-cdk-lib/aws-iam'; import * as cr from 'aws-cdk-lib/custom-resources'; import * as path from 'path'; -import * as nodejs from 'aws-cdk-lib/aws-lambda-nodejs'; export class PostgresLambdaStack extends cdk.Stack { constructor(scope: Construct, id: string, props?: cdk.StackProps) { @@ -31,10 +30,24 @@ export class PostgresLambdaStack extends cdk.Stack { credentials: rds.Credentials.fromGeneratedSecret('postgres'), }); - // Create a Lambda function that calls PostgreSQL using NodejsFunction - const lambdaToPostgres = new nodejs.NodejsFunction(this, 'LambdaToPostgres', { + // Create a Lambda function that calls PostgreSQL with Docker bundling + const lambdaToPostgres = new lambda.Function(this, 'LambdaToPostgres', { runtime: lambda.Runtime.NODEJS_LATEST, - entry: path.join(__dirname, '../lambda/lambda-to-postgres/index.js'), + handler: 'index.handler', + code: lambda.Code.fromAsset(path.join(__dirname, '../lambda/lambda-to-postgres'), { + bundling: { + image: cdk.DockerImage.fromRegistry('public.ecr.aws/sam/build-nodejs18.x'), + command: [ + 'bash', '-c', [ + 'cp -r . /tmp', + 'cd /tmp', + 'npm init -y', + 'npm install pg', + 'cp -r . /asset-output/' + ].join(' && ') + ], + }, + }), vpc, vpcSubnets: { subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS, @@ -43,11 +56,6 @@ export class PostgresLambdaStack extends cdk.Stack { DB_SECRET_ARN: dbCluster.secret?.secretArn || '', DB_NAME: 'demodb', }, - bundling: { - externalModules: [ - 'aws-sdk', // Use the AWS SDK available in the Lambda runtime - ], - }, timeout: cdk.Duration.seconds(30), }); @@ -57,21 +65,18 @@ export class PostgresLambdaStack extends cdk.Stack { // Grant the Lambda function permission to read the database secret dbCluster.secret?.grantRead(lambdaToPostgres); - // Create a Lambda function that is called by PostgreSQL using NodejsFunction - const postgresFunction = new nodejs.NodejsFunction(this, 'PostgresFunction', { + // Create a Lambda function that is called by PostgreSQL + const postgresFunction = new lambda.Function(this, 'PostgresFunction', { runtime: lambda.Runtime.NODEJS_LATEST, - entry: path.join(__dirname, '../lambda/postgres-to-lambda/index.js'), + handler: 'index.handler', + code: lambda.Code.fromAsset(path.join(__dirname, '../lambda/postgres-to-lambda')), environment: { FUNCTION_NAME: 'PostgresFunction', }, - bundling: { - externalModules: [ - 'aws-sdk', // Use the AWS SDK available in the Lambda runtime - ], - }, timeout: cdk.Duration.seconds(30), }); + // Create a role for PostgreSQL to assume to invoke Lambda const postgresLambdaRole = new iam.Role(this, 'PostgresLambdaRole', { assumedBy: new iam.ServicePrincipal('rds.amazonaws.com'), @@ -79,6 +84,7 @@ export class PostgresLambdaStack extends cdk.Stack { postgresFunction.grantInvoke(postgresLambdaRole); + const l1DbCluster = dbCluster.node.defaultChild as rds.CfnDBCluster const exisitingProperty = (l1DbCluster.associatedRoles as []) || []; console.log(exisitingProperty); @@ -92,10 +98,24 @@ export class PostgresLambdaStack extends cdk.Stack { l1DbCluster.addPropertyOverride('AssociatedRoles', updatedRoles); - // Create Lambda function for PostgreSQL setup using NodejsFunction - const setupFunction = new nodejs.NodejsFunction(this, 'PostgresSetupFunction', { + // Create Lambda function for PostgreSQL setup with Docker bundling + const setupFunction = new lambda.Function(this, 'PostgresSetupFunction', { runtime: lambda.Runtime.NODEJS_LATEST, - entry: path.join(__dirname, '../lambda/postgres-setup/index.js'), + handler: 'index.handler', + code: lambda.Code.fromAsset(path.join(__dirname, '../lambda/postgres-setup'), { + bundling: { + image: cdk.DockerImage.fromRegistry('public.ecr.aws/sam/build-nodejs18.x'), + command: [ + 'bash', '-c', [ + 'cp -r . /tmp', + 'cd /tmp', + 'npm init -y', + 'npm install pg', + 'cp -r . /asset-output/' + ].join(' && ') + ], + }, + }), vpc, vpcSubnets: { subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS, @@ -105,11 +125,6 @@ export class PostgresLambdaStack extends cdk.Stack { DB_NAME: 'demodb', POSTGRES_FUNCTION_NAME: postgresFunction.functionName, }, - bundling: { - externalModules: [ - 'aws-sdk', // Use the AWS SDK available in the Lambda runtime - ], - }, timeout: cdk.Duration.minutes(5), }); From e879b7145ab645bb1dd01d17cd1250b7432fde84 Mon Sep 17 00:00:00 2001 From: Michael Kaiser Date: Fri, 11 Jul 2025 12:49:24 -0500 Subject: [PATCH 09/23] Add user: '1000' to bundling parameters for Lambda functions --- .../lib/postgres-lambda-stack.ts | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/typescript/postgres-lambda/lib/postgres-lambda-stack.ts b/typescript/postgres-lambda/lib/postgres-lambda-stack.ts index 61b38f9782..b4ad3f2d5e 100644 --- a/typescript/postgres-lambda/lib/postgres-lambda-stack.ts +++ b/typescript/postgres-lambda/lib/postgres-lambda-stack.ts @@ -46,6 +46,7 @@ export class PostgresLambdaStack extends cdk.Stack { 'cp -r . /asset-output/' ].join(' && ') ], + user: '1000', }, }), vpc, @@ -69,7 +70,20 @@ export class PostgresLambdaStack extends cdk.Stack { const postgresFunction = new lambda.Function(this, 'PostgresFunction', { runtime: lambda.Runtime.NODEJS_LATEST, handler: 'index.handler', - code: lambda.Code.fromAsset(path.join(__dirname, '../lambda/postgres-to-lambda')), + code: lambda.Code.fromAsset(path.join(__dirname, '../lambda/postgres-to-lambda'), { + bundling: { + image: cdk.DockerImage.fromRegistry('public.ecr.aws/sam/build-nodejs18.x'), + command: [ + 'bash', '-c', [ + 'cp -r . /tmp', + 'cd /tmp', + 'npm init -y', + 'cp -r . /asset-output/' + ].join(' && ') + ], + user: '1000', + }, + }), environment: { FUNCTION_NAME: 'PostgresFunction', }, @@ -114,6 +128,7 @@ export class PostgresLambdaStack extends cdk.Stack { 'cp -r . /asset-output/' ].join(' && ') ], + user: '1000', }, }), vpc, From f4e657b5f74a136fc0e5315a83e04e056b0684c8 Mon Sep 17 00:00:00 2001 From: Michael Kaiser Date: Fri, 11 Jul 2025 13:00:10 -0500 Subject: [PATCH 10/23] Update code --- typescript/postgres-lambda/.gitignore | 9 +- .../lambda/lambda-to-postgres/index.js | 78 ++++++++++++++ .../lambda/lambda-to-postgres/package.json | 9 +- .../lambda/postgres-setup/index.js | 97 +++++++++++++++++ .../lambda/postgres-setup/package.json | 11 +- .../lambda/postgres-to-lambda/index.js | 102 ++++++++++++++++++ .../lib/postgres-lambda-stack.ts | 11 -- typescript/postgres-lambda/package.json | 4 +- 8 files changed, 285 insertions(+), 36 deletions(-) create mode 100644 typescript/postgres-lambda/lambda/lambda-to-postgres/index.js create mode 100644 typescript/postgres-lambda/lambda/postgres-setup/index.js create mode 100644 typescript/postgres-lambda/lambda/postgres-to-lambda/index.js diff --git a/typescript/postgres-lambda/.gitignore b/typescript/postgres-lambda/.gitignore index abb73b4410..8d1b397a0a 100644 --- a/typescript/postgres-lambda/.gitignore +++ b/typescript/postgres-lambda/.gitignore @@ -2,8 +2,8 @@ !jest.config.js *.d.ts node_modules -# Allow node_modules in Lambda directories -!lambda/*/node_modules +**/node_modules + !lambda/*/*.js # CDK asset staging directory @@ -12,11 +12,6 @@ cdk.out # Yarn specific .yarn/* -!.yarn/patches -!.yarn/plugins -!.yarn/releases -!.yarn/sdks -!.yarn/versions .pnp.* yarn-debug.log* yarn-error.log* diff --git a/typescript/postgres-lambda/lambda/lambda-to-postgres/index.js b/typescript/postgres-lambda/lambda/lambda-to-postgres/index.js new file mode 100644 index 0000000000..879eefbc1c --- /dev/null +++ b/typescript/postgres-lambda/lambda/lambda-to-postgres/index.js @@ -0,0 +1,78 @@ +const { Client } = require('pg'); +const { SecretsManager } = require('@aws-sdk/client-secrets-manager'); + +const secretsManager = new SecretsManager(); + +/** + * Lambda function that connects to PostgreSQL and executes a query + */ +exports.handler = async (event) => { + console.log('Event received:', JSON.stringify(event)); + + try { + // Get database credentials from Secrets Manager + const secretArn = process.env.DB_SECRET_ARN; + const dbName = process.env.DB_NAME; + + console.log(`Retrieving secret from ${secretArn}`); + const secretResponse = await secretsManager.getSecretValue({ SecretId: secretArn }); + const secret = JSON.parse(secretResponse.SecretString); + + // Create PostgreSQL client + const client = new Client({ + host: secret.host, + port: secret.port, + database: dbName, + user: secret.username, + password: secret.password, + ssl: { + rejectUnauthorized: false, // For demo purposes only, consider proper SSL setup in production + }, + connectionTimeoutMillis: 5000, + }); + + // Connect to the database + console.log('Connecting to PostgreSQL database...'); + await client.connect(); + + // Check if our demo table exists, if not create it + console.log('Creating demo table if it does not exist...'); + await client.query(` + CREATE TABLE IF NOT EXISTS demo_table ( + id SERIAL PRIMARY KEY, + message TEXT NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + `); + + // Insert a record + const message = event.message || 'Hello from Lambda!'; + console.log(`Inserting message: ${message}`); + await client.query('INSERT INTO demo_table (message) VALUES ($1)', [message]); + + // Query the records + console.log('Querying records...'); + const result = await client.query('SELECT * FROM demo_table ORDER BY created_at DESC LIMIT 10'); + + // Close the connection + await client.end(); + + // Return the results + return { + statusCode: 200, + body: JSON.stringify({ + message: 'Query executed successfully', + records: result.rows, + }), + }; + } catch (error) { + console.error('Error:', error); + return { + statusCode: 500, + body: JSON.stringify({ + message: 'Error executing query', + error: error.message, + }), + }; + } +}; diff --git a/typescript/postgres-lambda/lambda/lambda-to-postgres/package.json b/typescript/postgres-lambda/lambda/lambda-to-postgres/package.json index 80a154e90b..eadd465cd2 100644 --- a/typescript/postgres-lambda/lambda/lambda-to-postgres/package.json +++ b/typescript/postgres-lambda/lambda/lambda-to-postgres/package.json @@ -1,14 +1,9 @@ { - "name": "@lambda/lambda-to-postgres", + "name": "lambda-to-postgres", "version": "1.0.0", "description": "Lambda function that connects to PostgreSQL", "main": "index.js", - "scripts": { - "build": "echo 'No build needed for JavaScript files'", - "test": "echo \"Error: no test specified\" && exit 1" - }, "dependencies": { - "pg": "^8.16.3", - "aws-sdk": "^2.1377.0" + "pg": "^8.11.0" } } diff --git a/typescript/postgres-lambda/lambda/postgres-setup/index.js b/typescript/postgres-lambda/lambda/postgres-setup/index.js new file mode 100644 index 0000000000..ec65b52edf --- /dev/null +++ b/typescript/postgres-lambda/lambda/postgres-setup/index.js @@ -0,0 +1,97 @@ +const { Client } = require('pg'); +const { SecretsManagerClient, GetSecretValueCommand } = require('@aws-sdk/client-secrets-manager'); + +const secretsManager = new SecretsManagerClient(); + +exports.handler = async (event) => { + console.log('Event:', JSON.stringify(event, null, 2)); + + if (event.RequestType === 'Delete') { + return sendResponse(event, 'SUCCESS', 'Delete operation completed'); + } + + try { + const { DB_SECRET_ARN, DB_NAME, POSTGRES_FUNCTION_NAME, AWS_REGION } = process.env; + + // Get database credentials + const secretResponse = await secretsManager.send( + new GetSecretValueCommand({ SecretId: DB_SECRET_ARN }) + ); + const secret = JSON.parse(secretResponse.SecretString); + + // Connect to PostgreSQL + const client = new Client({ + host: secret.host, + port: secret.port, + database: DB_NAME, + user: secret.username, + password: secret.password, + ssl: { rejectUnauthorized: false } + }); + + await client.connect(); + + // Execute setup SQL + const setupSQL = ` + CREATE EXTENSION IF NOT EXISTS aws_lambda CASCADE; + + CREATE OR REPLACE FUNCTION process_data(data JSONB) + RETURNS JSONB AS $$ + SELECT payload FROM aws_lambda.invoke( + aws_commons.create_lambda_function_arn('${POSTGRES_FUNCTION_NAME}', '${AWS_REGION}'), + json_build_object('action', 'process', 'data', data)::text, + 'Event' + ); + $$ LANGUAGE SQL; + + CREATE OR REPLACE FUNCTION transform_data(data JSONB) + RETURNS JSONB AS $$ + SELECT payload FROM aws_lambda.invoke( + aws_commons.create_lambda_function_arn('${POSTGRES_FUNCTION_NAME}', '${AWS_REGION}'), + json_build_object('action', 'transform', 'data', data)::text, + 'Event' + ); + $$ LANGUAGE SQL; + + CREATE OR REPLACE FUNCTION validate_data(data JSONB) + RETURNS JSONB AS $$ + SELECT payload FROM aws_lambda.invoke( + aws_commons.create_lambda_function_arn('${POSTGRES_FUNCTION_NAME}', '${AWS_REGION}'), + json_build_object('action', 'validate', 'data', data)::text, + 'Event' + ); + $$ LANGUAGE SQL; + `; + + await client.query(setupSQL); + await client.end(); + + return sendResponse(event, 'SUCCESS', 'PostgreSQL setup completed successfully'); + + } catch (error) { + console.error('Error:', error); + return sendResponse(event, 'FAILED', error.message); + } +}; + +async function sendResponse(event, status, reason) { + const response = { + Status: status, + Reason: reason, + PhysicalResourceId: 'postgres-setup-' + Date.now(), + StackId: event.StackId, + RequestId: event.RequestId, + LogicalResourceId: event.LogicalResourceId + }; + + console.log('Response:', JSON.stringify(response, null, 2)); + + const fetch = (await import('node-fetch')).default; + await fetch(event.ResponseURL, { + method: 'PUT', + headers: { 'Content-Type': '' }, + body: JSON.stringify(response) + }); + + return response; +} diff --git a/typescript/postgres-lambda/lambda/postgres-setup/package.json b/typescript/postgres-lambda/lambda/postgres-setup/package.json index 27b675102b..c6ffd021b5 100644 --- a/typescript/postgres-lambda/lambda/postgres-setup/package.json +++ b/typescript/postgres-lambda/lambda/postgres-setup/package.json @@ -1,14 +1,9 @@ { - "name": "@lambda/postgres-setup", + "name": "postgres-setup", "version": "1.0.0", - "description": "Lambda function for automated PostgreSQL setup", + "description": "Lambda function for PostgreSQL setup", "main": "index.js", - "scripts": { - "build": "echo 'No build needed for JavaScript files'", - "test": "echo \"Error: no test specified\" && exit 1" - }, "dependencies": { - "pg": "^8.16.3", - "aws-sdk": "^2.1377.0" + "pg": "^8.11.0" } } diff --git a/typescript/postgres-lambda/lambda/postgres-to-lambda/index.js b/typescript/postgres-lambda/lambda/postgres-to-lambda/index.js new file mode 100644 index 0000000000..56b9eacc51 --- /dev/null +++ b/typescript/postgres-lambda/lambda/postgres-to-lambda/index.js @@ -0,0 +1,102 @@ +/** + * Lambda function that is called by PostgreSQL + * + * This function can be invoked from PostgreSQL using the aws_lambda extension + * Example SQL: + * + * SELECT * FROM aws_lambda.invoke( + * aws_commons.create_lambda_function_arn('PostgresFunction', 'us-east-1'), + * '{"action": "process", "data": {"id": 123, "value": "test"}}', + * 'Event' + * ); + */ +exports.handler = async (event) => { + console.log('Event received from PostgreSQL:', JSON.stringify(event)); + + try { + // Process the event data + const action = event.action || 'default'; + const data = event.data || {}; + + let result; + + // Perform different actions based on the event + switch (action) { + case 'process': + result = processData(data); + break; + case 'transform': + result = transformData(data); + break; + case 'validate': + result = validateData(data); + break; + default: + result = { status: 'success', message: 'Default action performed', data }; + } + + return { + statusCode: 200, + body: result, + }; + } catch (error) { + console.error('Error:', error); + return { + statusCode: 500, + body: { + status: 'error', + message: error.message, + }, + }; + } +}; + +/** + * Process data from PostgreSQL + */ +function processData(data) { + console.log('Processing data:', data); + return { + status: 'success', + message: 'Data processed successfully', + processedData: { + ...data, + processed: true, + timestamp: new Date().toISOString(), + }, + }; +} + +/** + * Transform data from PostgreSQL + */ +function transformData(data) { + console.log('Transforming data:', data); + return { + status: 'success', + message: 'Data transformed successfully', + transformedData: { + ...data, + transformed: true, + uppercase: data.value ? data.value.toUpperCase() : null, + timestamp: new Date().toISOString(), + }, + }; +} + +/** + * Validate data from PostgreSQL + */ +function validateData(data) { + console.log('Validating data:', data); + const isValid = data.id && data.value; + return { + status: isValid ? 'success' : 'error', + message: isValid ? 'Data is valid' : 'Data is invalid', + validationResult: { + isValid, + missingFields: !data.id ? ['id'] : !data.value ? ['value'] : [], + timestamp: new Date().toISOString(), + }, + }; +} diff --git a/typescript/postgres-lambda/lib/postgres-lambda-stack.ts b/typescript/postgres-lambda/lib/postgres-lambda-stack.ts index b4ad3f2d5e..eec4538f96 100644 --- a/typescript/postgres-lambda/lib/postgres-lambda-stack.ts +++ b/typescript/postgres-lambda/lib/postgres-lambda-stack.ts @@ -39,10 +39,6 @@ export class PostgresLambdaStack extends cdk.Stack { image: cdk.DockerImage.fromRegistry('public.ecr.aws/sam/build-nodejs18.x'), command: [ 'bash', '-c', [ - 'cp -r . /tmp', - 'cd /tmp', - 'npm init -y', - 'npm install pg', 'cp -r . /asset-output/' ].join(' && ') ], @@ -75,9 +71,6 @@ export class PostgresLambdaStack extends cdk.Stack { image: cdk.DockerImage.fromRegistry('public.ecr.aws/sam/build-nodejs18.x'), command: [ 'bash', '-c', [ - 'cp -r . /tmp', - 'cd /tmp', - 'npm init -y', 'cp -r . /asset-output/' ].join(' && ') ], @@ -121,10 +114,6 @@ export class PostgresLambdaStack extends cdk.Stack { image: cdk.DockerImage.fromRegistry('public.ecr.aws/sam/build-nodejs18.x'), command: [ 'bash', '-c', [ - 'cp -r . /tmp', - 'cd /tmp', - 'npm init -y', - 'npm install pg', 'cp -r . /asset-output/' ].join(' && ') ], diff --git a/typescript/postgres-lambda/package.json b/typescript/postgres-lambda/package.json index a6a303b0f6..48311266e4 100644 --- a/typescript/postgres-lambda/package.json +++ b/typescript/postgres-lambda/package.json @@ -6,9 +6,7 @@ ".", "lambda/*" ], - "bin": { - "postgres-lambda": "bin/postgres-lambda.js" - }, + "bin": "bin/postgres-lambda.js", "scripts": { "build": "tsc && yarn workspaces foreach -v -A run build", "build:lambda": "yarn workspaces foreach -v -A run build", From 2fab975cfb43c733b212c8b6a4e4b622278444f9 Mon Sep 17 00:00:00 2001 From: Michael Kaiser Date: Fri, 11 Jul 2025 14:53:48 -0500 Subject: [PATCH 11/23] fix: replace fetch with native https module in postgres-setup Lambda function --- .../lambda/postgres-setup/index.js | 64 +++++++++++++------ .../lambda/postgres-setup/package.json | 6 +- 2 files changed, 50 insertions(+), 20 deletions(-) diff --git a/typescript/postgres-lambda/lambda/postgres-setup/index.js b/typescript/postgres-lambda/lambda/postgres-setup/index.js index ec65b52edf..e61f007466 100644 --- a/typescript/postgres-lambda/lambda/postgres-setup/index.js +++ b/typescript/postgres-lambda/lambda/postgres-setup/index.js @@ -1,5 +1,7 @@ const { Client } = require('pg'); const { SecretsManagerClient, GetSecretValueCommand } = require('@aws-sdk/client-secrets-manager'); +const https = require('https'); +const url = require('url'); const secretsManager = new SecretsManagerClient(); @@ -74,24 +76,48 @@ exports.handler = async (event) => { } }; -async function sendResponse(event, status, reason) { - const response = { - Status: status, - Reason: reason, - PhysicalResourceId: 'postgres-setup-' + Date.now(), - StackId: event.StackId, - RequestId: event.RequestId, - LogicalResourceId: event.LogicalResourceId - }; - - console.log('Response:', JSON.stringify(response, null, 2)); - - const fetch = (await import('node-fetch')).default; - await fetch(event.ResponseURL, { - method: 'PUT', - headers: { 'Content-Type': '' }, - body: JSON.stringify(response) - }); +function sendResponse(event, status, reason) { + return new Promise((resolve, reject) => { + const responseBody = JSON.stringify({ + Status: status, + Reason: reason, + PhysicalResourceId: 'postgres-setup-' + Date.now(), + StackId: event.StackId, + RequestId: event.RequestId, + LogicalResourceId: event.LogicalResourceId, + Data: {} + }); + + console.log('Response:', responseBody); + + // Parse the URL + const parsedUrl = url.parse(event.ResponseURL); + + // Prepare the request options + const options = { + hostname: parsedUrl.hostname, + port: 443, + path: parsedUrl.path, + method: 'PUT', + headers: { + 'Content-Type': '', + 'Content-Length': responseBody.length + } + }; + + // Send the response + const request = https.request(options, (response) => { + console.log(`Status code: ${response.statusCode}`); + resolve({ status, reason }); + }); - return response; + request.on('error', (error) => { + console.error('Error sending response:', error); + reject(error); + }); + + // Write the response body and end the request + request.write(responseBody); + request.end(); + }); } diff --git a/typescript/postgres-lambda/lambda/postgres-setup/package.json b/typescript/postgres-lambda/lambda/postgres-setup/package.json index c6ffd021b5..214db2d514 100644 --- a/typescript/postgres-lambda/lambda/postgres-setup/package.json +++ b/typescript/postgres-lambda/lambda/postgres-setup/package.json @@ -3,7 +3,11 @@ "version": "1.0.0", "description": "Lambda function for PostgreSQL setup", "main": "index.js", + "scripts": { + "build": "mkdir -p node_modules && npm install --no-package-lock" + }, "dependencies": { - "pg": "^8.11.0" + "pg": "^8.11.0", + "@aws-sdk/client-secrets-manager": "^3.350.0" } } From 5ed3e573caa58c28d99e4bdef5ba8a8a6da0281b Mon Sep 17 00:00:00 2001 From: Michael Kaiser Date: Fri, 11 Jul 2025 14:53:57 -0500 Subject: [PATCH 12/23] refactor: standardize Lambda bundling configuration --- .../lib/postgres-lambda-stack.ts | 33 +++++++------------ 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/typescript/postgres-lambda/lib/postgres-lambda-stack.ts b/typescript/postgres-lambda/lib/postgres-lambda-stack.ts index eec4538f96..b1336f6b68 100644 --- a/typescript/postgres-lambda/lib/postgres-lambda-stack.ts +++ b/typescript/postgres-lambda/lib/postgres-lambda-stack.ts @@ -30,19 +30,20 @@ export class PostgresLambdaStack extends cdk.Stack { credentials: rds.Credentials.fromGeneratedSecret('postgres'), }); + const bundleCommand = [ + 'bash', '-c', [ + 'cp -r . /asset-output/', + ].join(' && ') + ] + // Create a Lambda function that calls PostgreSQL with Docker bundling const lambdaToPostgres = new lambda.Function(this, 'LambdaToPostgres', { runtime: lambda.Runtime.NODEJS_LATEST, handler: 'index.handler', code: lambda.Code.fromAsset(path.join(__dirname, '../lambda/lambda-to-postgres'), { bundling: { - image: cdk.DockerImage.fromRegistry('public.ecr.aws/sam/build-nodejs18.x'), - command: [ - 'bash', '-c', [ - 'cp -r . /asset-output/' - ].join(' && ') - ], - user: '1000', + image: lambda.Runtime.NODEJS_LATEST.bundlingImage, + command: bundleCommand, }, }), vpc, @@ -68,13 +69,8 @@ export class PostgresLambdaStack extends cdk.Stack { handler: 'index.handler', code: lambda.Code.fromAsset(path.join(__dirname, '../lambda/postgres-to-lambda'), { bundling: { - image: cdk.DockerImage.fromRegistry('public.ecr.aws/sam/build-nodejs18.x'), - command: [ - 'bash', '-c', [ - 'cp -r . /asset-output/' - ].join(' && ') - ], - user: '1000', + image: lambda.Runtime.NODEJS_LATEST.bundlingImage, + command: bundleCommand, }, }), environment: { @@ -111,13 +107,8 @@ export class PostgresLambdaStack extends cdk.Stack { handler: 'index.handler', code: lambda.Code.fromAsset(path.join(__dirname, '../lambda/postgres-setup'), { bundling: { - image: cdk.DockerImage.fromRegistry('public.ecr.aws/sam/build-nodejs18.x'), - command: [ - 'bash', '-c', [ - 'cp -r . /asset-output/' - ].join(' && ') - ], - user: '1000', + image: lambda.Runtime.NODEJS_LATEST.bundlingImage, + command: bundleCommand, }, }), vpc, From 09ffb67bce768d9758cfcc723b4aaf873c2f05af Mon Sep 17 00:00:00 2001 From: Michael Kaiser Date: Fri, 11 Jul 2025 14:54:06 -0500 Subject: [PATCH 13/23] chore: update Lambda package.json files for build scripts --- .../postgres-lambda/lambda/lambda-to-postgres/package.json | 3 +++ .../postgres-lambda/lambda/postgres-to-lambda/package.json | 4 ---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/typescript/postgres-lambda/lambda/lambda-to-postgres/package.json b/typescript/postgres-lambda/lambda/lambda-to-postgres/package.json index eadd465cd2..d05ebc7eab 100644 --- a/typescript/postgres-lambda/lambda/lambda-to-postgres/package.json +++ b/typescript/postgres-lambda/lambda/lambda-to-postgres/package.json @@ -3,6 +3,9 @@ "version": "1.0.0", "description": "Lambda function that connects to PostgreSQL", "main": "index.js", + "scripts": { + "build": "cd $(pwd) && npm install --no-package-lock" + }, "dependencies": { "pg": "^8.11.0" } diff --git a/typescript/postgres-lambda/lambda/postgres-to-lambda/package.json b/typescript/postgres-lambda/lambda/postgres-to-lambda/package.json index 62eebc805f..0cf7944a4c 100644 --- a/typescript/postgres-lambda/lambda/postgres-to-lambda/package.json +++ b/typescript/postgres-lambda/lambda/postgres-to-lambda/package.json @@ -3,10 +3,6 @@ "version": "1.0.0", "description": "Lambda function called by PostgreSQL", "main": "index.js", - "scripts": { - "build": "echo 'No build needed for JavaScript files'", - "test": "echo \"Error: no test specified\" && exit 1" - }, "dependencies": { "aws-sdk": "^2.1377.0" } From a74bdeaafb09088cc3cc9f465bf4e213e3c45459 Mon Sep 17 00:00:00 2001 From: Michael Kaiser Date: Fri, 11 Jul 2025 14:54:18 -0500 Subject: [PATCH 14/23] docs: update README.md and project configuration for Yarn workspaces --- typescript/postgres-lambda/.gitignore | 2 +- typescript/postgres-lambda/README.md | 13 ++++++++++--- typescript/postgres-lambda/package.json | 6 ++++-- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/typescript/postgres-lambda/.gitignore b/typescript/postgres-lambda/.gitignore index 8d1b397a0a..631f754fcd 100644 --- a/typescript/postgres-lambda/.gitignore +++ b/typescript/postgres-lambda/.gitignore @@ -2,7 +2,7 @@ !jest.config.js *.d.ts node_modules -**/node_modules +#**/node_modules !lambda/*/*.js diff --git a/typescript/postgres-lambda/README.md b/typescript/postgres-lambda/README.md index a5ca0640d0..3bf88313cb 100644 --- a/typescript/postgres-lambda/README.md +++ b/typescript/postgres-lambda/README.md @@ -144,6 +144,7 @@ SELECT validate_data('{"id": 789, "value": "valid data"}'::JSONB); ├── test/ # Unit tests ├── setup-postgres-lambda.sql # Reference SQL (now automated) ├── test-lambda.sh # Lambda testing script +├── .yarn/ # Yarn 2+ configuration └── README.md # This file ``` @@ -155,11 +156,11 @@ This project uses Yarn Workspaces to manage multiple packages in a monorepo stru # List all workspaces yarn workspaces list -# Run a command in all Lambda workspaces -yarn workspaces foreach -v --include '@lambda/*' run +# Run a command in all workspaces +yarn workspaces foreach -v -A run # Run a command in a specific workspace -yarn workspace @lambda/lambda-to-postgres run +yarn workspace postgres-to-lambda run # Install dependencies for all workspaces yarn install @@ -240,6 +241,12 @@ Before using in production: ### Useful Commands ```bash +# Build all packages +yarn build + +# Build only Lambda functions +yarn build:lambda + # Build and watch for changes yarn watch diff --git a/typescript/postgres-lambda/package.json b/typescript/postgres-lambda/package.json index 48311266e4..49f3a60c3f 100644 --- a/typescript/postgres-lambda/package.json +++ b/typescript/postgres-lambda/package.json @@ -12,7 +12,8 @@ "build:lambda": "yarn workspaces foreach -v -A run build", "watch": "tsc -w", "test": "jest", - "cdk": "cdk" + "cdk": "cdk", + "install-deps": "cd lambda/lambda-to-postgres && npm install --prefix ./deps && cd ../../lambda/postgres-to-lambda && npm install --prefix ./deps && cd ../../lambda/postgres-setup && npm install --prefix ./deps" }, "devDependencies": { "@types/jest": "^29.5.14", @@ -28,5 +29,6 @@ "aws-cdk-lib": "^2.204.0", "constructs": "^10.4.2", "esbuild": "^0.25.6" - } + }, + "packageManager": "yarn@4.9.2" } From af5ac03d263fc0bd1872e0078fa5dcbd1dcac15a Mon Sep 17 00:00:00 2001 From: Michael Kaiser Date: Fri, 11 Jul 2025 14:54:41 -0500 Subject: [PATCH 15/23] chore: add .yarnrc.yml for Yarn configuration --- typescript/postgres-lambda/.yarnrc.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 typescript/postgres-lambda/.yarnrc.yml diff --git a/typescript/postgres-lambda/.yarnrc.yml b/typescript/postgres-lambda/.yarnrc.yml new file mode 100644 index 0000000000..27a419b8b1 --- /dev/null +++ b/typescript/postgres-lambda/.yarnrc.yml @@ -0,0 +1,4 @@ +nodeLinker: node-modules +nmHoistingLimits: workspaces + +yarnPath: .yarn/releases/yarn-4.9.2.cjs From 9afb6bf2013a2ed52e93e0e56b535dded82ec32b Mon Sep 17 00:00:00 2001 From: Michael Kaiser Date: Fri, 11 Jul 2025 15:58:01 -0500 Subject: [PATCH 16/23] Fix PostgreSQL Lambda integration issues and improve setup process - Fix security group rule direction (allowDefaultPortFrom instead of allowDefaultPortTo) - Improve Lambda to PostgreSQL connection with increased timeout and better logging - Replace custom CloudFormation response handling with cfn-response library - Update PostgreSQL setup function to use standard CloudFormation response pattern - Add proper error handling in setup function --- .../lambda/lambda-to-postgres/index.js | 22 ++++--- .../lambda/postgres-setup/index.js | 62 +++---------------- .../lambda/postgres-setup/package.json | 5 +- .../lib/postgres-lambda-stack.ts | 2 +- 4 files changed, 26 insertions(+), 65 deletions(-) diff --git a/typescript/postgres-lambda/lambda/lambda-to-postgres/index.js b/typescript/postgres-lambda/lambda/lambda-to-postgres/index.js index 879eefbc1c..08bf34b670 100644 --- a/typescript/postgres-lambda/lambda/lambda-to-postgres/index.js +++ b/typescript/postgres-lambda/lambda/lambda-to-postgres/index.js @@ -8,16 +8,18 @@ const secretsManager = new SecretsManager(); */ exports.handler = async (event) => { console.log('Event received:', JSON.stringify(event)); - + try { // Get database credentials from Secrets Manager const secretArn = process.env.DB_SECRET_ARN; const dbName = process.env.DB_NAME; - + console.log(`Retrieving secret from ${secretArn}`); const secretResponse = await secretsManager.getSecretValue({ SecretId: secretArn }); const secret = JSON.parse(secretResponse.SecretString); - + const logSecret = {...secret, password: '*********'}; + console.log(logSecret); + // Create PostgreSQL client const client = new Client({ host: secret.host, @@ -28,13 +30,13 @@ exports.handler = async (event) => { ssl: { rejectUnauthorized: false, // For demo purposes only, consider proper SSL setup in production }, - connectionTimeoutMillis: 5000, + connectionTimeoutMillis: 10000, }); - + // Connect to the database console.log('Connecting to PostgreSQL database...'); await client.connect(); - + // Check if our demo table exists, if not create it console.log('Creating demo table if it does not exist...'); await client.query(` @@ -44,19 +46,19 @@ exports.handler = async (event) => { created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) `); - + // Insert a record const message = event.message || 'Hello from Lambda!'; console.log(`Inserting message: ${message}`); await client.query('INSERT INTO demo_table (message) VALUES ($1)', [message]); - + // Query the records console.log('Querying records...'); const result = await client.query('SELECT * FROM demo_table ORDER BY created_at DESC LIMIT 10'); - + // Close the connection await client.end(); - + // Return the results return { statusCode: 200, diff --git a/typescript/postgres-lambda/lambda/postgres-setup/index.js b/typescript/postgres-lambda/lambda/postgres-setup/index.js index e61f007466..32b52883aa 100644 --- a/typescript/postgres-lambda/lambda/postgres-setup/index.js +++ b/typescript/postgres-lambda/lambda/postgres-setup/index.js @@ -1,15 +1,14 @@ const { Client } = require('pg'); const { SecretsManagerClient, GetSecretValueCommand } = require('@aws-sdk/client-secrets-manager'); -const https = require('https'); -const url = require('url'); +const response = require('cfn-response'); const secretsManager = new SecretsManagerClient(); -exports.handler = async (event) => { +exports.handler = async (event, context) => { console.log('Event:', JSON.stringify(event, null, 2)); if (event.RequestType === 'Delete') { - return sendResponse(event, 'SUCCESS', 'Delete operation completed'); + return response.send(event, context, response.SUCCESS, {}, 'postgres-setup-delete'); } try { @@ -68,56 +67,15 @@ exports.handler = async (event) => { await client.query(setupSQL); await client.end(); - return sendResponse(event, 'SUCCESS', 'PostgreSQL setup completed successfully'); + // Send success response + return response.send(event, context, response.SUCCESS, { + Message: 'PostgreSQL setup completed successfully' + }, 'postgres-setup-' + Date.now()); } catch (error) { console.error('Error:', error); - return sendResponse(event, 'FAILED', error.message); + return response.send(event, context, response.FAILED, { + Error: error.message + }); } }; - -function sendResponse(event, status, reason) { - return new Promise((resolve, reject) => { - const responseBody = JSON.stringify({ - Status: status, - Reason: reason, - PhysicalResourceId: 'postgres-setup-' + Date.now(), - StackId: event.StackId, - RequestId: event.RequestId, - LogicalResourceId: event.LogicalResourceId, - Data: {} - }); - - console.log('Response:', responseBody); - - // Parse the URL - const parsedUrl = url.parse(event.ResponseURL); - - // Prepare the request options - const options = { - hostname: parsedUrl.hostname, - port: 443, - path: parsedUrl.path, - method: 'PUT', - headers: { - 'Content-Type': '', - 'Content-Length': responseBody.length - } - }; - - // Send the response - const request = https.request(options, (response) => { - console.log(`Status code: ${response.statusCode}`); - resolve({ status, reason }); - }); - - request.on('error', (error) => { - console.error('Error sending response:', error); - reject(error); - }); - - // Write the response body and end the request - request.write(responseBody); - request.end(); - }); -} diff --git a/typescript/postgres-lambda/lambda/postgres-setup/package.json b/typescript/postgres-lambda/lambda/postgres-setup/package.json index 214db2d514..2ea43720bf 100644 --- a/typescript/postgres-lambda/lambda/postgres-setup/package.json +++ b/typescript/postgres-lambda/lambda/postgres-setup/package.json @@ -7,7 +7,8 @@ "build": "mkdir -p node_modules && npm install --no-package-lock" }, "dependencies": { - "pg": "^8.11.0", - "@aws-sdk/client-secrets-manager": "^3.350.0" + "@aws-sdk/client-secrets-manager": "^3.350.0", + "cfn-response": "^1.0.1", + "pg": "^8.11.0" } } diff --git a/typescript/postgres-lambda/lib/postgres-lambda-stack.ts b/typescript/postgres-lambda/lib/postgres-lambda-stack.ts index b1336f6b68..34e76129ba 100644 --- a/typescript/postgres-lambda/lib/postgres-lambda-stack.ts +++ b/typescript/postgres-lambda/lib/postgres-lambda-stack.ts @@ -58,7 +58,7 @@ export class PostgresLambdaStack extends cdk.Stack { }); // Grant Lambda access to the DB - dbCluster.connections.allowDefaultPortTo(lambdaToPostgres); + dbCluster.connections.allowDefaultPortFrom(lambdaToPostgres); // Grant the Lambda function permission to read the database secret dbCluster.secret?.grantRead(lambdaToPostgres); From b35779182cd1ca50942604da54e8cbc6e3e80a26 Mon Sep 17 00:00:00 2001 From: Michael Kaiser Date: Fri, 11 Jul 2025 18:38:09 -0500 Subject: [PATCH 17/23] Improve PostgreSQL Lambda integration - Update SQL functions to use JSONB casting instead of text for proper JSON handling - Change Lambda invocation type from 'Event' to 'RequestResponse' for synchronous execution - Add parameter group to RDS cluster for custom configuration - Update Aurora PostgreSQL version to 17.4 - Fix code formatting and whitespace in Lambda functions --- .../lambda/postgres-setup/index.js | 12 ++++++------ .../lambda/postgres-to-lambda/index.js | 12 ++++++------ .../lib/postgres-lambda-stack.ts | 18 ++++++++++++++---- 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/typescript/postgres-lambda/lambda/postgres-setup/index.js b/typescript/postgres-lambda/lambda/postgres-setup/index.js index 32b52883aa..6a99ef4dc2 100644 --- a/typescript/postgres-lambda/lambda/postgres-setup/index.js +++ b/typescript/postgres-lambda/lambda/postgres-setup/index.js @@ -40,8 +40,8 @@ exports.handler = async (event, context) => { RETURNS JSONB AS $$ SELECT payload FROM aws_lambda.invoke( aws_commons.create_lambda_function_arn('${POSTGRES_FUNCTION_NAME}', '${AWS_REGION}'), - json_build_object('action', 'process', 'data', data)::text, - 'Event' + json_build_object('action', 'process', 'data', data)::JSONB, + 'RequestResponse' ); $$ LANGUAGE SQL; @@ -49,8 +49,8 @@ exports.handler = async (event, context) => { RETURNS JSONB AS $$ SELECT payload FROM aws_lambda.invoke( aws_commons.create_lambda_function_arn('${POSTGRES_FUNCTION_NAME}', '${AWS_REGION}'), - json_build_object('action', 'transform', 'data', data)::text, - 'Event' + json_build_object('action', 'transform', 'data', data)::JSONB, + 'RequestResponse' ); $$ LANGUAGE SQL; @@ -58,8 +58,8 @@ exports.handler = async (event, context) => { RETURNS JSONB AS $$ SELECT payload FROM aws_lambda.invoke( aws_commons.create_lambda_function_arn('${POSTGRES_FUNCTION_NAME}', '${AWS_REGION}'), - json_build_object('action', 'validate', 'data', data)::text, - 'Event' + json_build_object('action', 'validate', 'data', data)::JSONB, + 'RequestResponse' ); $$ LANGUAGE SQL; `; diff --git a/typescript/postgres-lambda/lambda/postgres-to-lambda/index.js b/typescript/postgres-lambda/lambda/postgres-to-lambda/index.js index 56b9eacc51..caf5326234 100644 --- a/typescript/postgres-lambda/lambda/postgres-to-lambda/index.js +++ b/typescript/postgres-lambda/lambda/postgres-to-lambda/index.js @@ -1,9 +1,9 @@ /** * Lambda function that is called by PostgreSQL - * + * * This function can be invoked from PostgreSQL using the aws_lambda extension * Example SQL: - * + * * SELECT * FROM aws_lambda.invoke( * aws_commons.create_lambda_function_arn('PostgresFunction', 'us-east-1'), * '{"action": "process", "data": {"id": 123, "value": "test"}}', @@ -12,14 +12,14 @@ */ exports.handler = async (event) => { console.log('Event received from PostgreSQL:', JSON.stringify(event)); - + try { // Process the event data const action = event.action || 'default'; const data = event.data || {}; - + let result; - + // Perform different actions based on the event switch (action) { case 'process': @@ -34,7 +34,7 @@ exports.handler = async (event) => { default: result = { status: 'success', message: 'Default action performed', data }; } - + return { statusCode: 200, body: result, diff --git a/typescript/postgres-lambda/lib/postgres-lambda-stack.ts b/typescript/postgres-lambda/lib/postgres-lambda-stack.ts index 34e76129ba..163e081dca 100644 --- a/typescript/postgres-lambda/lib/postgres-lambda-stack.ts +++ b/typescript/postgres-lambda/lib/postgres-lambda-stack.ts @@ -16,6 +16,13 @@ export class PostgresLambdaStack extends cdk.Stack { maxAzs: 2, natGateways: 1, }); + const parameterGroup = new rds.ParameterGroup(this, 'PGClusterParamGroup', { + engine: rds.DatabaseClusterEngine.auroraPostgres({ + version: rds.AuroraPostgresEngineVersion.VER_17_4 + }), + parameters: { + }, + }); // Create a PostgreSQL Aurora Serverless v2 cluster const dbCluster = new rds.DatabaseCluster(this, 'PostgresCluster', { @@ -28,13 +35,16 @@ export class PostgresLambdaStack extends cdk.Stack { serverlessV2MaxCapacity: 1, defaultDatabaseName: 'demodb', credentials: rds.Credentials.fromGeneratedSecret('postgres'), + parameterGroup: parameterGroup, }); + + const bundleCommand = [ - 'bash', '-c', [ - 'cp -r . /asset-output/', - ].join(' && ') - ] + 'bash', '-c', [ + 'cp -r . /asset-output/', + ].join(' && ') + ] // Create a Lambda function that calls PostgreSQL with Docker bundling const lambdaToPostgres = new lambda.Function(this, 'LambdaToPostgres', { From 12ac58d1457366d339283567818b5b78ef58f80b Mon Sep 17 00:00:00 2001 From: Michael Kaiser Date: Fri, 11 Jul 2025 20:00:38 -0500 Subject: [PATCH 18/23] Add comprehensive logging to PostgreSQL setup function - Add detailed logging throughout the setup process - Include environment variable logging for better debugging - Add verification step to confirm function creation - Improve error handling with stack traces - Add comments to SQL script for better readability - Include more detailed CloudFormation response data - Add connection timeout for better error handling --- .../lambda/postgres-setup/index.js | 85 ++++++++++++++++--- 1 file changed, 75 insertions(+), 10 deletions(-) diff --git a/typescript/postgres-lambda/lambda/postgres-setup/index.js b/typescript/postgres-lambda/lambda/postgres-setup/index.js index 6a99ef4dc2..0e520c9e2a 100644 --- a/typescript/postgres-lambda/lambda/postgres-setup/index.js +++ b/typescript/postgres-lambda/lambda/postgres-setup/index.js @@ -4,38 +4,78 @@ const response = require('cfn-response'); const secretsManager = new SecretsManagerClient(); +/** + * Lambda function to set up PostgreSQL with AWS Lambda integration + * This function is called by a CloudFormation Custom Resource + */ exports.handler = async (event, context) => { - console.log('Event:', JSON.stringify(event, null, 2)); + console.log('Event received:', JSON.stringify(event, null, 2)); + console.log('Context:', JSON.stringify({ + functionName: context.functionName, + functionVersion: context.functionVersion, + awsRequestId: context.awsRequestId, + logGroupName: context.logGroupName, + logStreamName: context.logStreamName, + remainingTimeMs: context.getRemainingTimeInMillis() + }, null, 2)); + // Handle delete event if (event.RequestType === 'Delete') { + console.log('Delete request received - no action needed for cleanup'); return response.send(event, context, response.SUCCESS, {}, 'postgres-setup-delete'); } try { + // Get environment variables const { DB_SECRET_ARN, DB_NAME, POSTGRES_FUNCTION_NAME, AWS_REGION } = process.env; + console.log('Environment variables:', JSON.stringify({ + DB_SECRET_ARN, + DB_NAME, + POSTGRES_FUNCTION_NAME, + AWS_REGION + }, null, 2)); - // Get database credentials + // Get database credentials from Secrets Manager + console.log(`Retrieving secret from ${DB_SECRET_ARN}`); const secretResponse = await secretsManager.send( new GetSecretValueCommand({ SecretId: DB_SECRET_ARN }) ); + + if (!secretResponse.SecretString) { + throw new Error('Secret string is empty or undefined'); + } + const secret = JSON.parse(secretResponse.SecretString); + console.log('Secret retrieved successfully. Host:', secret.host); - // Connect to PostgreSQL + // Create PostgreSQL client + console.log(`Connecting to PostgreSQL database ${DB_NAME} at ${secret.host}:${secret.port}`); const client = new Client({ host: secret.host, port: secret.port, database: DB_NAME, user: secret.username, password: secret.password, - ssl: { rejectUnauthorized: false } + ssl: { rejectUnauthorized: false }, + // Add connection timeout for better error handling + connectionTimeoutMillis: 10000 }); + // Connect to the database + console.log('Attempting to connect to PostgreSQL...'); await client.connect(); + console.log('Successfully connected to PostgreSQL'); - // Execute setup SQL + // Prepare setup SQL + console.log('Preparing SQL setup script'); + const lambdaArn = `aws_commons.create_lambda_function_arn('${POSTGRES_FUNCTION_NAME}', '${AWS_REGION}')`; + console.log(`Using Lambda ARN expression: ${lambdaArn}`); + const setupSQL = ` + -- Create aws_lambda extension if it doesn't exist CREATE EXTENSION IF NOT EXISTS aws_lambda CASCADE; - + + -- Create function to process data CREATE OR REPLACE FUNCTION process_data(data JSONB) RETURNS JSONB AS $$ SELECT payload FROM aws_lambda.invoke( @@ -45,6 +85,7 @@ exports.handler = async (event, context) => { ); $$ LANGUAGE SQL; + -- Create function to transform data CREATE OR REPLACE FUNCTION transform_data(data JSONB) RETURNS JSONB AS $$ SELECT payload FROM aws_lambda.invoke( @@ -54,6 +95,7 @@ exports.handler = async (event, context) => { ); $$ LANGUAGE SQL; + -- Create function to validate data CREATE OR REPLACE FUNCTION validate_data(data JSONB) RETURNS JSONB AS $$ SELECT payload FROM aws_lambda.invoke( @@ -64,18 +106,41 @@ exports.handler = async (event, context) => { $$ LANGUAGE SQL; `; - await client.query(setupSQL); + // Execute setup SQL + console.log('Executing SQL setup script...'); + const result = await client.query(setupSQL); + console.log('SQL setup script executed successfully:', JSON.stringify(result)); + + // Verify the functions were created + console.log('Verifying SQL functions were created...'); + const verifyResult = await client.query(` + SELECT proname, proargtypes, prosrc + FROM pg_proc + WHERE proname IN ('process_data', 'transform_data', 'validate_data') + `); + console.log(`Found ${verifyResult.rows.length} functions:`, JSON.stringify(verifyResult.rows.map(row => row.proname))); + + // Close the database connection + console.log('Closing PostgreSQL connection'); await client.end(); + console.log('PostgreSQL connection closed successfully'); // Send success response + console.log('Sending SUCCESS response to CloudFormation'); return response.send(event, context, response.SUCCESS, { - Message: 'PostgreSQL setup completed successfully' + Message: 'PostgreSQL setup completed successfully', + FunctionsCreated: verifyResult.rows.length, + LambdaFunction: POSTGRES_FUNCTION_NAME, + Region: AWS_REGION }, 'postgres-setup-' + Date.now()); } catch (error) { - console.error('Error:', error); + console.error('Error during PostgreSQL setup:', error); + console.error('Stack trace:', error.stack); return response.send(event, context, response.FAILED, { - Error: error.message + Error: error.message, + ErrorType: error.name, + StackTrace: error.stack }); } }; From 1b0c145f39791f314b189598bd8e3e8788d559f0 Mon Sep 17 00:00:00 2001 From: Michael Kaiser Date: Fri, 11 Jul 2025 20:10:18 -0500 Subject: [PATCH 19/23] Fix security group rule and add custom resource property - Fix security group rule direction for setup function (allowDefaultPortFrom instead of allowDefaultPortTo) - Add test property to custom resource to force update on deployment --- typescript/postgres-lambda/lib/postgres-lambda-stack.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/typescript/postgres-lambda/lib/postgres-lambda-stack.ts b/typescript/postgres-lambda/lib/postgres-lambda-stack.ts index 163e081dca..b9c1a90e40 100644 --- a/typescript/postgres-lambda/lib/postgres-lambda-stack.ts +++ b/typescript/postgres-lambda/lib/postgres-lambda-stack.ts @@ -134,7 +134,7 @@ export class PostgresLambdaStack extends cdk.Stack { }); // Grant setup function access to the DB and secrets - dbCluster.connections.allowDefaultPortTo(setupFunction); + dbCluster.connections.allowDefaultPortFrom(setupFunction); dbCluster.secret?.grantRead(setupFunction); // Create custom resource to trigger setup @@ -144,6 +144,9 @@ export class PostgresLambdaStack extends cdk.Stack { new cdk.CustomResource(this, 'PostgresSetupResource', { serviceToken: setupProvider.serviceToken, + properties: { + test: 'yo' + }, }); // Output the database endpoint and secret ARN From 5d1229074fbbf28412320b3ff0d1f175eb9b2e47 Mon Sep 17 00:00:00 2001 From: Michael Kaiser Date: Fri, 11 Jul 2025 20:12:25 -0500 Subject: [PATCH 20/23] Remove manual setup files that are now automated - Remove setup-postgres-lambda.sql as PostgreSQL setup is now automated via Lambda - Remove test-lambda.sh as testing can be done through the AWS Console or CLI directly - These files are no longer needed since the setup process is fully automated through CDK --- .../postgres-lambda/setup-postgres-lambda.sql | 41 ---------- typescript/postgres-lambda/test-lambda.sh | 75 ------------------- 2 files changed, 116 deletions(-) delete mode 100644 typescript/postgres-lambda/setup-postgres-lambda.sql delete mode 100755 typescript/postgres-lambda/test-lambda.sh diff --git a/typescript/postgres-lambda/setup-postgres-lambda.sql b/typescript/postgres-lambda/setup-postgres-lambda.sql deleted file mode 100644 index 97ecd8b36d..0000000000 --- a/typescript/postgres-lambda/setup-postgres-lambda.sql +++ /dev/null @@ -1,41 +0,0 @@ --- SQL script to set up PostgreSQL to call Lambda --- Replace with your AWS region (e.g., us-east-1) --- Replace with the name of your Lambda function - --- Create the aws_lambda extension -CREATE EXTENSION IF NOT EXISTS aws_lambda CASCADE; - --- Create a function to process data using Lambda -CREATE OR REPLACE FUNCTION process_data(data JSONB) -RETURNS JSONB AS $$ -SELECT payload FROM aws_lambda.invoke( - aws_commons.create_lambda_function_arn('', ''), - json_build_object('action', 'process', 'data', data)::text, - 'Event' -); -$$ LANGUAGE SQL; - --- Create a function to transform data using Lambda -CREATE OR REPLACE FUNCTION transform_data(data JSONB) -RETURNS JSONB AS $$ -SELECT payload FROM aws_lambda.invoke( - aws_commons.create_lambda_function_arn('', ''), - json_build_object('action', 'transform', 'data', data)::text, - 'Event' -); -$$ LANGUAGE SQL; - --- Create a function to validate data using Lambda -CREATE OR REPLACE FUNCTION validate_data(data JSONB) -RETURNS JSONB AS $$ -SELECT payload FROM aws_lambda.invoke( - aws_commons.create_lambda_function_arn('', ''), - json_build_object('action', 'validate', 'data', data)::text, - 'Event' -); -$$ LANGUAGE SQL; - --- Test the functions -SELECT process_data('{"id": 123, "value": "test"}'::JSONB); -SELECT transform_data('{"id": 456, "value": "hello world"}'::JSONB); -SELECT validate_data('{"id": 789, "value": "valid data"}'::JSONB); diff --git a/typescript/postgres-lambda/test-lambda.sh b/typescript/postgres-lambda/test-lambda.sh deleted file mode 100755 index 637b859ae4..0000000000 --- a/typescript/postgres-lambda/test-lambda.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/bash - -# Test script for PostgreSQL Lambda integration - -# Check if jq is installed -if ! command -v jq &> /dev/null; then - echo "Error: jq is required but not installed. Please install jq first." - exit 1 -fi - -# Function to display usage -function display_usage { - echo "Usage: $0 [options]" - echo "Options:" - echo " -f, --function-name NAME Lambda function name to test" - echo " -m, --message MESSAGE Message to send to Lambda (default: 'Test message')" - echo " -h, --help Display this help message" - exit 1 -} - -# Default values -MESSAGE="Test message" - -# Parse command line arguments -while [[ $# -gt 0 ]]; do - key="$1" - case $key in - -f|--function-name) - FUNCTION_NAME="$2" - shift - shift - ;; - -m|--message) - MESSAGE="$2" - shift - shift - ;; - -h|--help) - display_usage - ;; - *) - echo "Unknown option: $1" - display_usage - ;; - esac -done - -# Check if function name is provided -if [ -z "$FUNCTION_NAME" ]; then - echo "Error: Lambda function name is required." - display_usage -fi - -# Create payload -PAYLOAD=$(echo "{\"message\": \"$MESSAGE\"}" | jq -c .) - -# Create temporary file for response -RESPONSE_FILE=$(mktemp) - -echo "Invoking Lambda function: $FUNCTION_NAME" -echo "Payload: $PAYLOAD" - -# Invoke Lambda function -aws lambda invoke \ - --function-name "$FUNCTION_NAME" \ - --payload "$PAYLOAD" \ - --cli-binary-format raw-in-base64-out \ - "$RESPONSE_FILE" - -# Display response -echo "Response:" -cat "$RESPONSE_FILE" | jq . - -# Clean up -rm "$RESPONSE_FILE" From 368fd3becaaae1b921afde1937c275109afca64e Mon Sep 17 00:00:00 2001 From: Michael Kaiser Date: Fri, 11 Jul 2025 21:04:37 -0500 Subject: [PATCH 21/23] Clean up documentation and remove unused code - Update README.md to remove references to deleted files (setup-postgres-lambda.sql and test-lambda.sh) - Add information about synchronous Lambda invocation in PostgreSQL to Lambda flow - Remove unused parameter group from RDS cluster configuration - Remove test property from custom resource as it's not needed - Remove outdated SUMMARY.md file that's no longer relevant --- typescript/postgres-lambda/README.md | 14 ++---- typescript/postgres-lambda/SUMMARY.md | 50 ------------------- .../lib/postgres-lambda-stack.ts | 11 ---- 3 files changed, 4 insertions(+), 71 deletions(-) delete mode 100644 typescript/postgres-lambda/SUMMARY.md diff --git a/typescript/postgres-lambda/README.md b/typescript/postgres-lambda/README.md index 3bf88313cb..358e5c0ca3 100644 --- a/typescript/postgres-lambda/README.md +++ b/typescript/postgres-lambda/README.md @@ -80,12 +80,7 @@ No manual setup required! 🎉 ### Test Lambda → PostgreSQL -Using the provided test script: -```bash -./test-lambda.sh --function-name --message "Hello World" -``` - -Or using AWS CLI directly: +Using AWS CLI directly: ```bash aws lambda invoke \ --function-name \ @@ -129,8 +124,9 @@ SELECT validate_data('{"id": 789, "value": "valid data"}'::JSONB); 1. **Extension Setup**: Uses `aws_lambda` extension for Lambda invocation (automated) 2. **Function Creation**: SQL functions wrap Lambda calls with proper ARN construction (automated) -3. **Event Processing**: Lambda receives structured JSON events from PostgreSQL -4. **Result Return**: Lambda response becomes available in SQL query results +3. **Synchronous Invocation**: Uses 'RequestResponse' invocation type for immediate results +4. **Event Processing**: Lambda receives structured JSON events from PostgreSQL +5. **Result Return**: Lambda response becomes available in SQL query results ## Project Structure @@ -142,8 +138,6 @@ SELECT validate_data('{"id": 789, "value": "valid data"}'::JSONB); │ ├── postgres-to-lambda/ # Lambda called by PostgreSQL │ └── postgres-setup/ # Lambda for automated setup ├── test/ # Unit tests -├── setup-postgres-lambda.sql # Reference SQL (now automated) -├── test-lambda.sh # Lambda testing script ├── .yarn/ # Yarn 2+ configuration └── README.md # This file ``` diff --git a/typescript/postgres-lambda/SUMMARY.md b/typescript/postgres-lambda/SUMMARY.md deleted file mode 100644 index aab23f5e01..0000000000 --- a/typescript/postgres-lambda/SUMMARY.md +++ /dev/null @@ -1,50 +0,0 @@ -# PostgreSQL and Lambda Integration Example - Summary - -This CDK example demonstrates the integration between AWS Lambda and Aurora PostgreSQL Serverless v2. It showcases two key integration patterns: - -## 1. Lambda to PostgreSQL - -The first pattern demonstrates how a Lambda function can connect to and interact with a PostgreSQL database: - -- The Lambda function (`LambdaToPostgres`) retrieves database credentials from AWS Secrets Manager -- It establishes a connection to the PostgreSQL database -- It creates a table if it doesn't exist, inserts data, and queries the database -- The function returns the query results - -## 2. PostgreSQL to Lambda - -The second pattern demonstrates how PostgreSQL can invoke a Lambda function: - -- PostgreSQL uses the `aws_lambda` extension to call Lambda functions -- The Lambda function (`PostgresFunction`) receives data from PostgreSQL -- It processes the data based on the action specified in the event -- It returns results that can be used in SQL queries - -## Security Features - -The example implements several security best practices: - -- The database is deployed in a private subnet -- Security groups restrict access to the database -- Credentials are stored in AWS Secrets Manager -- IAM roles limit permissions to only what's necessary - -## Helper Scripts - -The example includes several helper scripts: - -- `test-lambda.sh`: For testing the Lambda functions -- `connect-to-postgres.sh`: For connecting to the PostgreSQL database -- `setup-postgres-lambda.sql`: For setting up the PostgreSQL database to call Lambda - -## Deployment - -The example can be deployed with standard CDK commands: - -```bash -npm install -npm run build -npx cdk deploy -``` - -After deployment, users need to set up the PostgreSQL database to call Lambda by creating the `aws_lambda` extension and defining functions that invoke Lambda. diff --git a/typescript/postgres-lambda/lib/postgres-lambda-stack.ts b/typescript/postgres-lambda/lib/postgres-lambda-stack.ts index b9c1a90e40..8611aec429 100644 --- a/typescript/postgres-lambda/lib/postgres-lambda-stack.ts +++ b/typescript/postgres-lambda/lib/postgres-lambda-stack.ts @@ -16,13 +16,6 @@ export class PostgresLambdaStack extends cdk.Stack { maxAzs: 2, natGateways: 1, }); - const parameterGroup = new rds.ParameterGroup(this, 'PGClusterParamGroup', { - engine: rds.DatabaseClusterEngine.auroraPostgres({ - version: rds.AuroraPostgresEngineVersion.VER_17_4 - }), - parameters: { - }, - }); // Create a PostgreSQL Aurora Serverless v2 cluster const dbCluster = new rds.DatabaseCluster(this, 'PostgresCluster', { @@ -35,7 +28,6 @@ export class PostgresLambdaStack extends cdk.Stack { serverlessV2MaxCapacity: 1, defaultDatabaseName: 'demodb', credentials: rds.Credentials.fromGeneratedSecret('postgres'), - parameterGroup: parameterGroup, }); @@ -144,9 +136,6 @@ export class PostgresLambdaStack extends cdk.Stack { new cdk.CustomResource(this, 'PostgresSetupResource', { serviceToken: setupProvider.serviceToken, - properties: { - test: 'yo' - }, }); // Output the database endpoint and secret ARN From f37763351db4df9acb5d04373466595a71066756 Mon Sep 17 00:00:00 2001 From: Michael Kaiser Date: Mon, 14 Jul 2025 11:09:39 -0500 Subject: [PATCH 22/23] Go back to classic yarn --- typescript/postgres-lambda/.yarnrc.yml | 4 ---- typescript/postgres-lambda/package.json | 16 ++++++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) delete mode 100644 typescript/postgres-lambda/.yarnrc.yml diff --git a/typescript/postgres-lambda/.yarnrc.yml b/typescript/postgres-lambda/.yarnrc.yml deleted file mode 100644 index 27a419b8b1..0000000000 --- a/typescript/postgres-lambda/.yarnrc.yml +++ /dev/null @@ -1,4 +0,0 @@ -nodeLinker: node-modules -nmHoistingLimits: workspaces - -yarnPath: .yarn/releases/yarn-4.9.2.cjs diff --git a/typescript/postgres-lambda/package.json b/typescript/postgres-lambda/package.json index 49f3a60c3f..09b8920ef7 100644 --- a/typescript/postgres-lambda/package.json +++ b/typescript/postgres-lambda/package.json @@ -2,10 +2,15 @@ "name": "postgres-lambda", "version": "0.1.0", "private": true, - "workspaces": [ - ".", - "lambda/*" - ], + "workspaces": { + "packages": [ + "lambda/*" + ], + "nohoist": [ + "**", + "**/**" + ] + }, "bin": "bin/postgres-lambda.js", "scripts": { "build": "tsc && yarn workspaces foreach -v -A run build", @@ -29,6 +34,5 @@ "aws-cdk-lib": "^2.204.0", "constructs": "^10.4.2", "esbuild": "^0.25.6" - }, - "packageManager": "yarn@4.9.2" + } } From 1f0b52100082b50cfd5cea767de2a8dd365d5930 Mon Sep 17 00:00:00 2001 From: Michael Kaiser Date: Mon, 14 Jul 2025 11:36:35 -0500 Subject: [PATCH 23/23] Fix package build scrits --- typescript/postgres-lambda/package.json | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/typescript/postgres-lambda/package.json b/typescript/postgres-lambda/package.json index 09b8920ef7..6e91ecf261 100644 --- a/typescript/postgres-lambda/package.json +++ b/typescript/postgres-lambda/package.json @@ -13,12 +13,10 @@ }, "bin": "bin/postgres-lambda.js", "scripts": { - "build": "tsc && yarn workspaces foreach -v -A run build", - "build:lambda": "yarn workspaces foreach -v -A run build", + "build": "tsc", "watch": "tsc -w", "test": "jest", - "cdk": "cdk", - "install-deps": "cd lambda/lambda-to-postgres && npm install --prefix ./deps && cd ../../lambda/postgres-to-lambda && npm install --prefix ./deps && cd ../../lambda/postgres-setup && npm install --prefix ./deps" + "cdk": "cdk" }, "devDependencies": { "@types/jest": "^29.5.14", @@ -30,7 +28,6 @@ "typescript": "~5.6.3" }, "dependencies": { - "@aws-cdk/aws-lambda-nodejs": "^1.203.0", "aws-cdk-lib": "^2.204.0", "constructs": "^10.4.2", "esbuild": "^0.25.6"