diff --git a/typescript/postgres-lambda/.gitignore b/typescript/postgres-lambda/.gitignore new file mode 100644 index 000000000..631f754fc --- /dev/null +++ b/typescript/postgres-lambda/.gitignore @@ -0,0 +1,32 @@ +*.js +!jest.config.js +*.d.ts +node_modules +#**/node_modules + +!lambda/*/*.js + +# CDK asset staging directory +.cdk.staging +cdk.out + +# Yarn specific +.yarn/* +.pnp.* +yarn-debug.log* +yarn-error.log* + +# Build artifacts +dist/ +build/ +*.tsbuildinfo + +# IDE files +.idea/ +.vscode/ +*.swp +*.swo + +# OS files +.DS_Store +Thumbs.db diff --git a/typescript/postgres-lambda/.npmignore b/typescript/postgres-lambda/.npmignore new file mode 100644 index 000000000..c1d6d45dc --- /dev/null +++ b/typescript/postgres-lambda/.npmignore @@ -0,0 +1,6 @@ +*.ts +!*.d.ts + +# CDK asset staging directory +.cdk.staging +cdk.out diff --git a/typescript/postgres-lambda/README.md b/typescript/postgres-lambda/README.md new file mode 100644 index 000000000..358e5c0ca --- /dev/null +++ b/typescript/postgres-lambda/README.md @@ -0,0 +1,295 @@ +# PostgreSQL and Lambda Integration Example + +A complete AWS CDK example demonstrating bidirectional integration between Aurora PostgreSQL Serverless v2 and AWS Lambda functions. + +## What This Example Demonstrates + +- **Lambda → PostgreSQL**: Lambda function that connects to and queries PostgreSQL +- **PostgreSQL → Lambda**: PostgreSQL database that invokes Lambda functions using the `aws_lambda` extension +- **Secure Architecture**: Private subnets, IAM roles, and Secrets Manager integration +- **Production-Ready**: Includes error handling, connection pooling, and security best practices +- **Automated Setup**: Custom CDK resource automatically configures PostgreSQL extensions and functions +- **Yarn Workspaces**: Organized monorepo structure for managing multiple Lambda functions + +## Architecture + +```mermaid +graph TD + subgraph VPC + subgraph "Private Subnet" + DB[Aurora PostgreSQL\nServerless v2] + L1[Lambda Function\nLambdaToPostgres] + L2[Lambda Function\nPostgresFunction] + L3[Lambda Function\nPostgresSetup] + end + end + + L1 -->|"1. Connect and Query"| DB + DB -->|"2. Invoke via aws_lambda extension"| L2 + L2 -->|"3. Return Result"| DB + L3 -->|"4. Setup Extensions & Functions"| DB + + SM[AWS Secrets Manager] -->|Provide Credentials| L1 + SM -->|Provide Credentials| L3 + + style DB fill:#E3F2FD,stroke:#1976D2,stroke-width:2px,color:#000 + style L1 fill:#FFF3E0,stroke:#F57C00,stroke-width:2px,color:#000 + style L2 fill:#FFF3E0,stroke:#F57C00,stroke-width:2px,color:#000 + style L3 fill:#E8F5E8,stroke:#4CAF50,stroke-width:2px,color:#000 + style SM fill:#F3E5F5,stroke:#7B1FA2,stroke-width:2px,color:#000 +``` + +**Components:** +- Aurora PostgreSQL Serverless v2 cluster (private subnet) +- Lambda function for database operations (`LambdaToPostgres`) +- Lambda function invokable from PostgreSQL (`PostgresFunction`) +- Lambda function for automated setup (`PostgresSetupFunction`) +- IAM roles with least-privilege permissions +- Security groups for network access control +- AWS Secrets Manager for credential storage +- Custom CDK resource for automated PostgreSQL configuration + +## Quick Start + +### Prerequisites + +- AWS CDK v2 installed (`npm install -g aws-cdk`) +- Node.js 18.x or later +- Yarn package manager installed +- AWS CLI configured with appropriate credentials + +### Deploy + +```bash +# Install dependencies using yarn workspaces +yarn install + +# Deploy the stack (setup is now automated!) +yarn cdk deploy +``` + +The deployment will automatically: +- Create the Aurora PostgreSQL cluster +- Deploy all Lambda functions +- Configure PostgreSQL extensions and functions +- Set up all necessary permissions + +No manual setup required! 🎉 + +## Testing + +### Test Lambda → PostgreSQL + +Using AWS CLI directly: +```bash +aws lambda invoke \ + --function-name \ + --payload '{"message": "Hello from CLI!"}' \ + response.json && cat response.json +``` + +### Test PostgreSQL → Lambda + +Connect to PostgreSQL and test the functions: +```bash +psql -h -U postgres -d demodb +``` + +```sql +-- Test the PostgreSQL to Lambda integration +SELECT process_data('{"id": 123, "value": "test"}'::JSONB); +SELECT transform_data('{"id": 456, "value": "hello world"}'::JSONB); +SELECT validate_data('{"id": 789, "value": "valid data"}'::JSONB); +``` + +## How It Works + +### Automated Setup Process + +1. **CDK Deployment**: Stack creates all resources including a setup Lambda function +2. **Custom Resource**: Triggers the setup Lambda after database is ready +3. **Extension Installation**: Setup function creates the `aws_lambda` extension +4. **Function Creation**: Creates SQL functions that wrap Lambda invocations +5. **Ready to Use**: Database is immediately ready for bidirectional Lambda integration + +### Lambda to PostgreSQL Flow + +1. **Credential Retrieval**: Function retrieves DB credentials from Secrets Manager +2. **Connection**: Establishes secure SSL connection to PostgreSQL +3. **Table Management**: Creates demo table if it doesn't exist +4. **Data Operations**: Inserts message and queries recent records +5. **Response**: Returns formatted results with error handling + +### PostgreSQL to Lambda Flow + +1. **Extension Setup**: Uses `aws_lambda` extension for Lambda invocation (automated) +2. **Function Creation**: SQL functions wrap Lambda calls with proper ARN construction (automated) +3. **Synchronous Invocation**: Uses 'RequestResponse' invocation type for immediate results +4. **Event Processing**: Lambda receives structured JSON events from PostgreSQL +5. **Result Return**: Lambda response becomes available in SQL query results + +## Project Structure + +``` +├── bin/ # CDK app entry point +├── lib/ # CDK stack definition +├── lambda/ # Lambda function source code +│ ├── lambda-to-postgres/ # Lambda that calls PostgreSQL +│ ├── postgres-to-lambda/ # Lambda called by PostgreSQL +│ └── postgres-setup/ # Lambda for automated setup +├── test/ # Unit tests +├── .yarn/ # Yarn 2+ configuration +└── README.md # This file +``` + +## Yarn Workspaces + +This project uses Yarn Workspaces to manage multiple packages in a monorepo structure: + +```bash +# List all workspaces +yarn workspaces list + +# Run a command in all workspaces +yarn workspaces foreach -v -A run + +# Run a command in a specific workspace +yarn workspace postgres-to-lambda run + +# Install dependencies for all workspaces +yarn install +``` + +The workspace structure allows for: +- Shared dependencies between packages +- Individual package management +- Simplified build and deployment process +- Better organization of Lambda functions + +## Configuration + +### Environment Variables + +The Lambda functions use these environment variables (set automatically by CDK): + +- `DB_SECRET_ARN`: ARN of the database credentials secret +- `DB_NAME`: Database name (default: `demodb`) +- `POSTGRES_FUNCTION_NAME`: Name of the Lambda function called by PostgreSQL +- `AWS_REGION`: AWS region for Lambda ARN construction + +### Customization + +- **Database Configuration**: Modify `lib/postgres-lambda-stack.ts` +- **Lambda Logic**: Update files in `lambda/` directories +- **Setup SQL**: Customize `lambda/postgres-setup/index.js` + +## Security Features + +✅ **Network Security** +- Database in private subnets +- Security groups with minimal required access +- No direct internet access to database + +✅ **Access Control** +- IAM roles with least-privilege permissions +- Secrets Manager for credential storage +- SSL/TLS encryption for database connections + +✅ **Monitoring** +- CloudWatch logs for all Lambda functions +- Database performance insights available +- VPC Flow Logs (can be enabled) + +## Production Considerations + +Before using in production: + +- [ ] Enable SSL certificate validation (`rejectUnauthorized: true`) +- [ ] Implement connection pooling (consider RDS Proxy) +- [ ] Set up proper monitoring and alerting +- [ ] Configure backup and disaster recovery +- [ ] Review and tighten IAM policies +- [ ] Enable database encryption at rest +- [ ] Set up VPC endpoints for AWS services +- [ ] Implement proper error handling and retry logic + +## Troubleshooting + +### Common Issues + +**Connection Timeouts** +- Check security group rules +- Verify Lambda is in correct VPC/subnets +- Confirm database is running + +**Permission Errors** +- Verify IAM roles have required permissions +- Check Secrets Manager access +- Confirm Lambda execution role + +**Setup Function Issues** +- Check CloudWatch logs for the PostgresSetupFunction +- Verify custom resource completed successfully +- Ensure database is accessible from setup Lambda + +### Useful Commands + +```bash +# Build all packages +yarn build + +# Build only Lambda functions +yarn build:lambda + +# Build and watch for changes +yarn watch + +# Run tests +yarn test + +# View CloudFormation template +yarn cdk synth + +# Compare deployed vs current state +yarn cdk diff + +# View stack outputs +aws cloudformation describe-stacks --stack-name PostgresLambdaStack --query 'Stacks[0].Outputs' + +# Check setup function logs +aws logs describe-log-groups --log-group-name-prefix /aws/lambda/PostgresLambdaStack-PostgresSetupFunction +``` + +## Cleanup + +```bash +yarn cdk destroy +``` + +**Note**: This will delete all resources including the database and any data stored in it. + +## Cost Optimization + +- Aurora Serverless v2 scales to zero when not in use +- Lambda functions only charge for execution time +- Setup function runs only once during deployment +- Consider Reserved Capacity for consistent workloads +- Monitor usage with AWS Cost Explorer + +## Related Examples + +- [Lambda with RDS Proxy](../lambda-rds-proxy/) +- [Aurora Serverless v1](../aurora-serverless-v1/) +- [PostgreSQL with CDK](../postgresql-cdk/) + +## Contributing + +1. Fork the repository +2. Create a feature branch +3. Make your changes +4. Add tests if applicable +5. Submit a pull request + +## License + +This example is provided under the MIT-0 License. See the LICENSE file for details. diff --git a/typescript/postgres-lambda/bin/postgres-lambda.ts b/typescript/postgres-lambda/bin/postgres-lambda.ts new file mode 100644 index 000000000..682eb13ce --- /dev/null +++ b/typescript/postgres-lambda/bin/postgres-lambda.ts @@ -0,0 +1,16 @@ +#!/usr/bin/env node +import * as cdk from 'aws-cdk-lib'; +import { PostgresLambdaStack } from '../lib/postgres-lambda-stack'; + +const app = new cdk.App(); +new PostgresLambdaStack(app, 'PostgresLambdaStack', { + /* If you don't specify 'env', this stack will be environment-agnostic. + * Account/Region-dependent features and context lookups will not work, + * but a single synthesized template can be deployed anywhere. */ + + /* Uncomment the next line to specialize this stack for the AWS Account + * and Region that are implied by the current CLI configuration. */ + env: { account: process.env.CDK_DEFAULT_ACCOUNT, region: process.env.CDK_DEFAULT_REGION }, + + /* For more information, see https://docs.aws.amazon.com/cdk/latest/guide/environments.html */ +}); \ No newline at end of file diff --git a/typescript/postgres-lambda/cdk.json b/typescript/postgres-lambda/cdk.json new file mode 100644 index 000000000..15397fccb --- /dev/null +++ b/typescript/postgres-lambda/cdk.json @@ -0,0 +1,98 @@ +{ + "app": "npx ts-node --prefer-ts-exts bin/postgres-lambda.ts", + "watch": { + "include": [ + "**" + ], + "exclude": [ + "README.md", + "cdk*.json", + "**/*.d.ts", + "**/*.js", + "tsconfig.json", + "package*.json", + "yarn.lock", + "node_modules", + "test" + ] + }, + "context": { + "@aws-cdk/aws-lambda:recognizeLayerVersion": true, + "@aws-cdk/core:checkSecretUsage": true, + "@aws-cdk/core:target-partitions": [ + "aws", + "aws-cn" + ], + "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, + "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, + "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true, + "@aws-cdk/aws-iam:minimizePolicies": true, + "@aws-cdk/core:validateSnapshotRemovalPolicy": true, + "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, + "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, + "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true, + "@aws-cdk/aws-apigateway:disableCloudWatchRole": true, + "@aws-cdk/core:enablePartitionLiterals": true, + "@aws-cdk/aws-events:eventsTargetQueueSameAccount": true, + "@aws-cdk/aws-ecs:disableExplicitDeploymentControllerForCircuitBreaker": true, + "@aws-cdk/aws-iam:importedRoleStackSafeDefaultPolicyName": true, + "@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true, + "@aws-cdk/aws-route53-patters:useCertificate": true, + "@aws-cdk/customresources:installLatestAwsSdkDefault": false, + "@aws-cdk/aws-rds:databaseProxyUniqueResourceName": true, + "@aws-cdk/aws-codedeploy:removeAlarmsFromDeploymentGroup": true, + "@aws-cdk/aws-apigateway:authorizerChangeDeploymentLogicalId": true, + "@aws-cdk/aws-ec2:launchTemplateDefaultUserData": true, + "@aws-cdk/aws-secretsmanager:useAttachedSecretResourcePolicyForSecretTargetAttachments": true, + "@aws-cdk/aws-redshift:columnId": true, + "@aws-cdk/aws-stepfunctions-tasks:enableEmrServicePolicyV2": true, + "@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": true, + "@aws-cdk/aws-apigateway:requestValidatorUniqueId": true, + "@aws-cdk/aws-kms:aliasNameRef": true, + "@aws-cdk/aws-kms:applyImportedAliasPermissionsToPrincipal": true, + "@aws-cdk/aws-autoscaling:generateLaunchTemplateInsteadOfLaunchConfig": true, + "@aws-cdk/core:includePrefixInUniqueNameGeneration": true, + "@aws-cdk/aws-efs:denyAnonymousAccess": true, + "@aws-cdk/aws-opensearchservice:enableOpensearchMultiAzWithStandby": true, + "@aws-cdk/aws-lambda-nodejs:useLatestRuntimeVersion": true, + "@aws-cdk/aws-efs:mountTargetOrderInsensitiveLogicalId": true, + "@aws-cdk/aws-rds:auroraClusterChangeScopeOfInstanceParameterGroupWithEachParameters": true, + "@aws-cdk/aws-appsync:useArnForSourceApiAssociationIdentifier": true, + "@aws-cdk/aws-rds:preventRenderingDeprecatedCredentials": true, + "@aws-cdk/aws-codepipeline-actions:useNewDefaultBranchForCodeCommitSource": true, + "@aws-cdk/aws-cloudwatch-actions:changeLambdaPermissionLogicalIdForLambdaAction": true, + "@aws-cdk/aws-codepipeline:crossAccountKeysDefaultValueToFalse": true, + "@aws-cdk/aws-codepipeline:defaultPipelineTypeToV2": true, + "@aws-cdk/aws-kms:reduceCrossAccountRegionPolicyScope": true, + "@aws-cdk/aws-eks:nodegroupNameAttribute": true, + "@aws-cdk/aws-ec2:ebsDefaultGp3Volume": true, + "@aws-cdk/aws-ecs:removeDefaultDeploymentAlarm": true, + "@aws-cdk/custom-resources:logApiResponseDataPropertyTrueDefault": false, + "@aws-cdk/aws-s3:keepNotificationInImportedBucket": false, + "@aws-cdk/aws-ecs:enableImdsBlockingDeprecatedFeature": false, + "@aws-cdk/aws-ecs:disableEcsImdsBlocking": true, + "@aws-cdk/aws-ecs:reduceEc2FargateCloudWatchPermissions": true, + "@aws-cdk/aws-dynamodb:resourcePolicyPerReplica": true, + "@aws-cdk/aws-ec2:ec2SumTImeoutEnabled": true, + "@aws-cdk/aws-appsync:appSyncGraphQLAPIScopeLambdaPermission": true, + "@aws-cdk/aws-rds:setCorrectValueForDatabaseInstanceReadReplicaInstanceResourceId": true, + "@aws-cdk/core:cfnIncludeRejectComplexResourceUpdateCreatePolicyIntrinsics": true, + "@aws-cdk/aws-lambda-nodejs:sdkV3ExcludeSmithyPackages": true, + "@aws-cdk/aws-stepfunctions-tasks:fixRunEcsTaskPolicy": true, + "@aws-cdk/aws-ec2:bastionHostUseAmazonLinux2023ByDefault": true, + "@aws-cdk/aws-route53-targets:userPoolDomainNameMethodWithoutCustomResource": true, + "@aws-cdk/aws-elasticloadbalancingV2:albDualstackWithoutPublicIpv4SecurityGroupRulesDefault": true, + "@aws-cdk/aws-iam:oidcRejectUnauthorizedConnections": true, + "@aws-cdk/core:enableAdditionalMetadataCollection": true, + "@aws-cdk/aws-lambda:createNewPoliciesWithAddToRolePolicy": false, + "@aws-cdk/aws-s3:setUniqueReplicationRoleName": true, + "@aws-cdk/aws-events:requireEventBusPolicySid": true, + "@aws-cdk/core:aspectPrioritiesMutating": true, + "@aws-cdk/aws-dynamodb:retainTableReplica": true, + "@aws-cdk/aws-stepfunctions:useDistributedMapResultWriterV2": true, + "@aws-cdk/s3-notifications:addS3TrustKeyPolicyForSnsSubscriptions": true, + "@aws-cdk/aws-ec2:requirePrivateSubnetsForEgressOnlyInternetGateway": true, + "@aws-cdk/aws-s3:publicAccessBlockedByDefault": true, + "@aws-cdk/aws-lambda:useCdkManagedLogGroup": true + } +} diff --git a/typescript/postgres-lambda/jest.config.js b/typescript/postgres-lambda/jest.config.js new file mode 100644 index 000000000..08263b895 --- /dev/null +++ b/typescript/postgres-lambda/jest.config.js @@ -0,0 +1,8 @@ +module.exports = { + testEnvironment: 'node', + roots: ['/test'], + testMatch: ['**/*.test.ts'], + transform: { + '^.+\\.tsx?$': 'ts-jest' + } +}; diff --git a/typescript/postgres-lambda/lambda/lambda-to-postgres/index.js b/typescript/postgres-lambda/lambda/lambda-to-postgres/index.js new file mode 100644 index 000000000..08bf34b67 --- /dev/null +++ b/typescript/postgres-lambda/lambda/lambda-to-postgres/index.js @@ -0,0 +1,80 @@ +const { Client } = require('pg'); +const { SecretsManager } = require('@aws-sdk/client-secrets-manager'); + +const secretsManager = new SecretsManager(); + +/** + * Lambda function that connects to PostgreSQL and executes a query + */ +exports.handler = async (event) => { + console.log('Event received:', JSON.stringify(event)); + + try { + // Get database credentials from Secrets Manager + const secretArn = process.env.DB_SECRET_ARN; + const dbName = process.env.DB_NAME; + + console.log(`Retrieving secret from ${secretArn}`); + const secretResponse = await secretsManager.getSecretValue({ SecretId: secretArn }); + const secret = JSON.parse(secretResponse.SecretString); + const logSecret = {...secret, password: '*********'}; + console.log(logSecret); + + // Create PostgreSQL client + const client = new Client({ + host: secret.host, + port: secret.port, + database: dbName, + user: secret.username, + password: secret.password, + ssl: { + rejectUnauthorized: false, // For demo purposes only, consider proper SSL setup in production + }, + connectionTimeoutMillis: 10000, + }); + + // Connect to the database + console.log('Connecting to PostgreSQL database...'); + await client.connect(); + + // Check if our demo table exists, if not create it + console.log('Creating demo table if it does not exist...'); + await client.query(` + CREATE TABLE IF NOT EXISTS demo_table ( + id SERIAL PRIMARY KEY, + message TEXT NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + `); + + // Insert a record + const message = event.message || 'Hello from Lambda!'; + console.log(`Inserting message: ${message}`); + await client.query('INSERT INTO demo_table (message) VALUES ($1)', [message]); + + // Query the records + console.log('Querying records...'); + const result = await client.query('SELECT * FROM demo_table ORDER BY created_at DESC LIMIT 10'); + + // Close the connection + await client.end(); + + // Return the results + return { + statusCode: 200, + body: JSON.stringify({ + message: 'Query executed successfully', + records: result.rows, + }), + }; + } catch (error) { + console.error('Error:', error); + return { + statusCode: 500, + body: JSON.stringify({ + message: 'Error executing query', + error: error.message, + }), + }; + } +}; diff --git a/typescript/postgres-lambda/lambda/lambda-to-postgres/package.json b/typescript/postgres-lambda/lambda/lambda-to-postgres/package.json new file mode 100644 index 000000000..d05ebc7ea --- /dev/null +++ b/typescript/postgres-lambda/lambda/lambda-to-postgres/package.json @@ -0,0 +1,12 @@ +{ + "name": "lambda-to-postgres", + "version": "1.0.0", + "description": "Lambda function that connects to PostgreSQL", + "main": "index.js", + "scripts": { + "build": "cd $(pwd) && npm install --no-package-lock" + }, + "dependencies": { + "pg": "^8.11.0" + } +} diff --git a/typescript/postgres-lambda/lambda/postgres-setup/index.js b/typescript/postgres-lambda/lambda/postgres-setup/index.js new file mode 100644 index 000000000..0e520c9e2 --- /dev/null +++ b/typescript/postgres-lambda/lambda/postgres-setup/index.js @@ -0,0 +1,146 @@ +const { Client } = require('pg'); +const { SecretsManagerClient, GetSecretValueCommand } = require('@aws-sdk/client-secrets-manager'); +const response = require('cfn-response'); + +const secretsManager = new SecretsManagerClient(); + +/** + * Lambda function to set up PostgreSQL with AWS Lambda integration + * This function is called by a CloudFormation Custom Resource + */ +exports.handler = async (event, context) => { + console.log('Event received:', JSON.stringify(event, null, 2)); + console.log('Context:', JSON.stringify({ + functionName: context.functionName, + functionVersion: context.functionVersion, + awsRequestId: context.awsRequestId, + logGroupName: context.logGroupName, + logStreamName: context.logStreamName, + remainingTimeMs: context.getRemainingTimeInMillis() + }, null, 2)); + + // Handle delete event + if (event.RequestType === 'Delete') { + console.log('Delete request received - no action needed for cleanup'); + return response.send(event, context, response.SUCCESS, {}, 'postgres-setup-delete'); + } + + try { + // Get environment variables + const { DB_SECRET_ARN, DB_NAME, POSTGRES_FUNCTION_NAME, AWS_REGION } = process.env; + console.log('Environment variables:', JSON.stringify({ + DB_SECRET_ARN, + DB_NAME, + POSTGRES_FUNCTION_NAME, + AWS_REGION + }, null, 2)); + + // Get database credentials from Secrets Manager + console.log(`Retrieving secret from ${DB_SECRET_ARN}`); + const secretResponse = await secretsManager.send( + new GetSecretValueCommand({ SecretId: DB_SECRET_ARN }) + ); + + if (!secretResponse.SecretString) { + throw new Error('Secret string is empty or undefined'); + } + + const secret = JSON.parse(secretResponse.SecretString); + console.log('Secret retrieved successfully. Host:', secret.host); + + // Create PostgreSQL client + console.log(`Connecting to PostgreSQL database ${DB_NAME} at ${secret.host}:${secret.port}`); + const client = new Client({ + host: secret.host, + port: secret.port, + database: DB_NAME, + user: secret.username, + password: secret.password, + ssl: { rejectUnauthorized: false }, + // Add connection timeout for better error handling + connectionTimeoutMillis: 10000 + }); + + // Connect to the database + console.log('Attempting to connect to PostgreSQL...'); + await client.connect(); + console.log('Successfully connected to PostgreSQL'); + + // Prepare setup SQL + console.log('Preparing SQL setup script'); + const lambdaArn = `aws_commons.create_lambda_function_arn('${POSTGRES_FUNCTION_NAME}', '${AWS_REGION}')`; + console.log(`Using Lambda ARN expression: ${lambdaArn}`); + + const setupSQL = ` + -- Create aws_lambda extension if it doesn't exist + CREATE EXTENSION IF NOT EXISTS aws_lambda CASCADE; + + -- Create function to process data + CREATE OR REPLACE FUNCTION process_data(data JSONB) + RETURNS JSONB AS $$ + SELECT payload FROM aws_lambda.invoke( + aws_commons.create_lambda_function_arn('${POSTGRES_FUNCTION_NAME}', '${AWS_REGION}'), + json_build_object('action', 'process', 'data', data)::JSONB, + 'RequestResponse' + ); + $$ LANGUAGE SQL; + + -- Create function to transform data + CREATE OR REPLACE FUNCTION transform_data(data JSONB) + RETURNS JSONB AS $$ + SELECT payload FROM aws_lambda.invoke( + aws_commons.create_lambda_function_arn('${POSTGRES_FUNCTION_NAME}', '${AWS_REGION}'), + json_build_object('action', 'transform', 'data', data)::JSONB, + 'RequestResponse' + ); + $$ LANGUAGE SQL; + + -- Create function to validate data + CREATE OR REPLACE FUNCTION validate_data(data JSONB) + RETURNS JSONB AS $$ + SELECT payload FROM aws_lambda.invoke( + aws_commons.create_lambda_function_arn('${POSTGRES_FUNCTION_NAME}', '${AWS_REGION}'), + json_build_object('action', 'validate', 'data', data)::JSONB, + 'RequestResponse' + ); + $$ LANGUAGE SQL; + `; + + // Execute setup SQL + console.log('Executing SQL setup script...'); + const result = await client.query(setupSQL); + console.log('SQL setup script executed successfully:', JSON.stringify(result)); + + // Verify the functions were created + console.log('Verifying SQL functions were created...'); + const verifyResult = await client.query(` + SELECT proname, proargtypes, prosrc + FROM pg_proc + WHERE proname IN ('process_data', 'transform_data', 'validate_data') + `); + console.log(`Found ${verifyResult.rows.length} functions:`, JSON.stringify(verifyResult.rows.map(row => row.proname))); + + // Close the database connection + console.log('Closing PostgreSQL connection'); + await client.end(); + console.log('PostgreSQL connection closed successfully'); + + // Send success response + console.log('Sending SUCCESS response to CloudFormation'); + return response.send(event, context, response.SUCCESS, { + Message: 'PostgreSQL setup completed successfully', + FunctionsCreated: verifyResult.rows.length, + LambdaFunction: POSTGRES_FUNCTION_NAME, + Region: AWS_REGION + }, 'postgres-setup-' + Date.now()); + + } catch (error) { + console.error('Error during PostgreSQL setup:', error); + console.error('Stack trace:', error.stack); + return response.send(event, context, response.FAILED, { + Error: error.message, + ErrorType: error.name, + StackTrace: error.stack + }); + } +}; diff --git a/typescript/postgres-lambda/lambda/postgres-setup/package.json b/typescript/postgres-lambda/lambda/postgres-setup/package.json new file mode 100644 index 000000000..2ea43720b --- /dev/null +++ b/typescript/postgres-lambda/lambda/postgres-setup/package.json @@ -0,0 +1,14 @@ +{ + "name": "postgres-setup", + "version": "1.0.0", + "description": "Lambda function for PostgreSQL setup", + "main": "index.js", + "scripts": { + "build": "mkdir -p node_modules && npm install --no-package-lock" + }, + "dependencies": { + "@aws-sdk/client-secrets-manager": "^3.350.0", + "cfn-response": "^1.0.1", + "pg": "^8.11.0" + } +} diff --git a/typescript/postgres-lambda/lambda/postgres-to-lambda/index.js b/typescript/postgres-lambda/lambda/postgres-to-lambda/index.js new file mode 100644 index 000000000..caf532623 --- /dev/null +++ b/typescript/postgres-lambda/lambda/postgres-to-lambda/index.js @@ -0,0 +1,102 @@ +/** + * Lambda function that is called by PostgreSQL + * + * This function can be invoked from PostgreSQL using the aws_lambda extension + * Example SQL: + * + * SELECT * FROM aws_lambda.invoke( + * aws_commons.create_lambda_function_arn('PostgresFunction', 'us-east-1'), + * '{"action": "process", "data": {"id": 123, "value": "test"}}', + * 'Event' + * ); + */ +exports.handler = async (event) => { + console.log('Event received from PostgreSQL:', JSON.stringify(event)); + + try { + // Process the event data + const action = event.action || 'default'; + const data = event.data || {}; + + let result; + + // Perform different actions based on the event + switch (action) { + case 'process': + result = processData(data); + break; + case 'transform': + result = transformData(data); + break; + case 'validate': + result = validateData(data); + break; + default: + result = { status: 'success', message: 'Default action performed', data }; + } + + return { + statusCode: 200, + body: result, + }; + } catch (error) { + console.error('Error:', error); + return { + statusCode: 500, + body: { + status: 'error', + message: error.message, + }, + }; + } +}; + +/** + * Process data from PostgreSQL + */ +function processData(data) { + console.log('Processing data:', data); + return { + status: 'success', + message: 'Data processed successfully', + processedData: { + ...data, + processed: true, + timestamp: new Date().toISOString(), + }, + }; +} + +/** + * Transform data from PostgreSQL + */ +function transformData(data) { + console.log('Transforming data:', data); + return { + status: 'success', + message: 'Data transformed successfully', + transformedData: { + ...data, + transformed: true, + uppercase: data.value ? data.value.toUpperCase() : null, + timestamp: new Date().toISOString(), + }, + }; +} + +/** + * Validate data from PostgreSQL + */ +function validateData(data) { + console.log('Validating data:', data); + const isValid = data.id && data.value; + return { + status: isValid ? 'success' : 'error', + message: isValid ? 'Data is valid' : 'Data is invalid', + validationResult: { + isValid, + missingFields: !data.id ? ['id'] : !data.value ? ['value'] : [], + timestamp: new Date().toISOString(), + }, + }; +} diff --git a/typescript/postgres-lambda/lambda/postgres-to-lambda/package.json b/typescript/postgres-lambda/lambda/postgres-to-lambda/package.json new file mode 100644 index 000000000..0cf7944a4 --- /dev/null +++ b/typescript/postgres-lambda/lambda/postgres-to-lambda/package.json @@ -0,0 +1,9 @@ +{ + "name": "@lambda/postgres-to-lambda", + "version": "1.0.0", + "description": "Lambda function called by PostgreSQL", + "main": "index.js", + "dependencies": { + "aws-sdk": "^2.1377.0" + } +} diff --git a/typescript/postgres-lambda/lib/postgres-lambda-stack.ts b/typescript/postgres-lambda/lib/postgres-lambda-stack.ts new file mode 100644 index 000000000..8611aec42 --- /dev/null +++ b/typescript/postgres-lambda/lib/postgres-lambda-stack.ts @@ -0,0 +1,167 @@ +import * as cdk from 'aws-cdk-lib'; +import { Construct } from 'constructs'; +import * as ec2 from 'aws-cdk-lib/aws-ec2'; +import * as rds from 'aws-cdk-lib/aws-rds'; +import * as lambda from 'aws-cdk-lib/aws-lambda'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import * as cr from 'aws-cdk-lib/custom-resources'; +import * as path from 'path'; + +export class PostgresLambdaStack extends cdk.Stack { + constructor(scope: Construct, id: string, props?: cdk.StackProps) { + super(scope, id, props); + + // Create a VPC for our application + const vpc = new ec2.Vpc(this, 'PostgresLambdaVpc', { + maxAzs: 2, + natGateways: 1, + }); + + // Create a PostgreSQL Aurora Serverless v2 cluster + const dbCluster = new rds.DatabaseCluster(this, 'PostgresCluster', { + engine: rds.DatabaseClusterEngine.auroraPostgres({ + version: rds.AuroraPostgresEngineVersion.VER_17_4, + }), + vpc: vpc, + writer: rds.ClusterInstance.serverlessV2('writer'), + serverlessV2MinCapacity: 0.5, + serverlessV2MaxCapacity: 1, + defaultDatabaseName: 'demodb', + credentials: rds.Credentials.fromGeneratedSecret('postgres'), + }); + + + + const bundleCommand = [ + 'bash', '-c', [ + 'cp -r . /asset-output/', + ].join(' && ') + ] + + // Create a Lambda function that calls PostgreSQL with Docker bundling + const lambdaToPostgres = new lambda.Function(this, 'LambdaToPostgres', { + runtime: lambda.Runtime.NODEJS_LATEST, + handler: 'index.handler', + code: lambda.Code.fromAsset(path.join(__dirname, '../lambda/lambda-to-postgres'), { + bundling: { + image: lambda.Runtime.NODEJS_LATEST.bundlingImage, + command: bundleCommand, + }, + }), + vpc, + vpcSubnets: { + subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS, + }, + environment: { + DB_SECRET_ARN: dbCluster.secret?.secretArn || '', + DB_NAME: 'demodb', + }, + timeout: cdk.Duration.seconds(30), + }); + + // Grant Lambda access to the DB + dbCluster.connections.allowDefaultPortFrom(lambdaToPostgres); + + // Grant the Lambda function permission to read the database secret + dbCluster.secret?.grantRead(lambdaToPostgres); + + // Create a Lambda function that is called by PostgreSQL + const postgresFunction = new lambda.Function(this, 'PostgresFunction', { + runtime: lambda.Runtime.NODEJS_LATEST, + handler: 'index.handler', + code: lambda.Code.fromAsset(path.join(__dirname, '../lambda/postgres-to-lambda'), { + bundling: { + image: lambda.Runtime.NODEJS_LATEST.bundlingImage, + command: bundleCommand, + }, + }), + environment: { + FUNCTION_NAME: 'PostgresFunction', + }, + timeout: cdk.Duration.seconds(30), + }); + + + // Create a role for PostgreSQL to assume to invoke Lambda + const postgresLambdaRole = new iam.Role(this, 'PostgresLambdaRole', { + assumedBy: new iam.ServicePrincipal('rds.amazonaws.com'), + }); + + postgresFunction.grantInvoke(postgresLambdaRole); + + + const l1DbCluster = dbCluster.node.defaultChild as rds.CfnDBCluster + const exisitingProperty = (l1DbCluster.associatedRoles as []) || []; + console.log(exisitingProperty); + + const newRole: { FeatureName: string, RoleArn: string } = { + FeatureName: "Lambda", // Changed to PascalCase + RoleArn: postgresLambdaRole.roleArn // Changed to PascalCase + }; + + const updatedRoles: { [key in 'featureName' | 'FeatureName' | 'roleArn' | 'RoleArn']?: string; }[] = [...exisitingProperty, newRole]; + + l1DbCluster.addPropertyOverride('AssociatedRoles', updatedRoles); + + // Create Lambda function for PostgreSQL setup with Docker bundling + const setupFunction = new lambda.Function(this, 'PostgresSetupFunction', { + runtime: lambda.Runtime.NODEJS_LATEST, + handler: 'index.handler', + code: lambda.Code.fromAsset(path.join(__dirname, '../lambda/postgres-setup'), { + bundling: { + image: lambda.Runtime.NODEJS_LATEST.bundlingImage, + command: bundleCommand, + }, + }), + vpc, + vpcSubnets: { + subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS, + }, + environment: { + DB_SECRET_ARN: dbCluster.secret?.secretArn || '', + DB_NAME: 'demodb', + POSTGRES_FUNCTION_NAME: postgresFunction.functionName, + }, + timeout: cdk.Duration.minutes(5), + }); + + // Grant setup function access to the DB and secrets + dbCluster.connections.allowDefaultPortFrom(setupFunction); + dbCluster.secret?.grantRead(setupFunction); + + // Create custom resource to trigger setup + const setupProvider = new cr.Provider(this, 'PostgresSetupProvider', { + onEventHandler: setupFunction, + }); + + new cdk.CustomResource(this, 'PostgresSetupResource', { + serviceToken: setupProvider.serviceToken, + }); + + // Output the database endpoint and secret ARN + new cdk.CfnOutput(this, 'DBClusterEndpoint', { + value: dbCluster.clusterEndpoint.hostname, + description: 'The endpoint of the database cluster', + }); + + new cdk.CfnOutput(this, 'DBSecretArn', { + value: dbCluster.secret?.secretArn || 'No secret created', + description: 'The ARN of the database credentials secret', + }); + + new cdk.CfnOutput(this, 'LambdaToPostgresFunctionName', { + value: lambdaToPostgres.functionName, + description: 'The name of the Lambda function that calls PostgreSQL', + }); + + new cdk.CfnOutput(this, 'PostgresFunctionName', { + value: postgresFunction.functionName, + description: 'The name of the Lambda function that is called by PostgreSQL', + }); + + new cdk.CfnOutput(this, 'PostgresLambdaRoleArn', { + value: postgresLambdaRole.roleArn, + description: 'The ARN of the role that PostgreSQL can assume to invoke Lambda', + }); + } +} diff --git a/typescript/postgres-lambda/package.json b/typescript/postgres-lambda/package.json new file mode 100644 index 000000000..6e91ecf26 --- /dev/null +++ b/typescript/postgres-lambda/package.json @@ -0,0 +1,35 @@ +{ + "name": "postgres-lambda", + "version": "0.1.0", + "private": true, + "workspaces": { + "packages": [ + "lambda/*" + ], + "nohoist": [ + "**", + "**/**" + ] + }, + "bin": "bin/postgres-lambda.js", + "scripts": { + "build": "tsc", + "watch": "tsc -w", + "test": "jest", + "cdk": "cdk" + }, + "devDependencies": { + "@types/jest": "^29.5.14", + "@types/node": "22.7.9", + "aws-cdk": "^2.1020.2", + "jest": "^29.7.0", + "ts-jest": "^29.2.5", + "ts-node": "^10.9.2", + "typescript": "~5.6.3" + }, + "dependencies": { + "aws-cdk-lib": "^2.204.0", + "constructs": "^10.4.2", + "esbuild": "^0.25.6" + } +} diff --git a/typescript/postgres-lambda/test/postgres-lambda.test.ts b/typescript/postgres-lambda/test/postgres-lambda.test.ts new file mode 100644 index 000000000..4c261f22e --- /dev/null +++ b/typescript/postgres-lambda/test/postgres-lambda.test.ts @@ -0,0 +1,17 @@ +// import * as cdk from 'aws-cdk-lib'; +// import { Template } from 'aws-cdk-lib/assertions'; +// import * as PostgresLambda from '../lib/postgres-lambda-stack'; + +// example test. To run these tests, uncomment this file along with the +// example resource in lib/postgres-lambda-stack.ts +test('SQS Queue Created', () => { +// const app = new cdk.App(); +// // WHEN +// const stack = new PostgresLambda.PostgresLambdaStack(app, 'MyTestStack'); +// // THEN +// const template = Template.fromStack(stack); + +// template.hasResourceProperties('AWS::SQS::Queue', { +// VisibilityTimeout: 300 +// }); +}); diff --git a/typescript/postgres-lambda/tsconfig.json b/typescript/postgres-lambda/tsconfig.json new file mode 100644 index 000000000..28bb557fa --- /dev/null +++ b/typescript/postgres-lambda/tsconfig.json @@ -0,0 +1,31 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": [ + "es2022" + ], + "declaration": true, + "strict": true, + "noImplicitAny": true, + "strictNullChecks": true, + "noImplicitThis": true, + "alwaysStrict": true, + "noUnusedLocals": false, + "noUnusedParameters": false, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": false, + "inlineSourceMap": true, + "inlineSources": true, + "experimentalDecorators": true, + "strictPropertyInitialization": false, + "typeRoots": [ + "./node_modules/@types" + ] + }, + "exclude": [ + "node_modules", + "cdk.out" + ] +}