|
| 1 | +{ |
| 2 | + "title": "Real-time Data Pipeline with Amazon Kinesis and AWS Lambda", |
| 3 | + "description": "Create a serverless real-time data pipeline using Amazon Kinesis, AWS Lambda, and Amazon DynamoDB", |
| 4 | + "language": ".NET", |
| 5 | + "level": "200", |
| 6 | + "framework": "CDK", |
| 7 | + "introBox": { |
| 8 | + "headline": "How it works", |
| 9 | + "text": [ |
| 10 | + "This pattern demonstrates how to create a serverless real-time data pipeline using Amazon Kinesis for data ingestion, AWS Lambda for processing, Amazon DynamoDB for data storage, and includes comprehensive error handling mechanisms.", |
| 11 | + "The data pipeline works as follows:", |
| 12 | + "1. Data is ingested through an Amazon Kinesis Data Stream, which acts as the entry point for real-time data.", |
| 13 | + "2. An AWS Lambda function is automatically triggered when new records appear in the Kinesis stream.", |
| 14 | + "3. The Lambda function processes the incoming data and stores it in an Amazon DynamoDB table.", |
| 15 | + "4. For robust error handling, the system includes several mechanisms:", |
| 16 | + " - Processing errors are logged to Amazon CloudWatch", |
| 17 | + " - Errors can be stored in a separate DynamoDB table for tracking", |
| 18 | + " - Failed records are automatically retried up to a configured maximum count", |
| 19 | + " - Persistently failing records can be sent to an Amazon SQS Dead Letter Queue (DLQ) for further investigation", |
| 20 | + "The entire infrastructure is defined and deployed using AWS CDK with .NET, including all necessary AWS resources, IAM roles, and permissions, ensuring a fully automated and reproducible deployment process." |
| 21 | + ] |
| 22 | + }, |
| 23 | + "gitHub": { |
| 24 | + "template": { |
| 25 | + "repoURL": "https://github.com/aws-samples/serverless-patterns/tree/main/kinesis-lambda-dynamodb-pipeline-dotnet-cdk", |
| 26 | + "templateURL": "serverless-patterns/kinesis-lambda-dynamodb-pipeline-dotnet-cdk", |
| 27 | + "projectFolder": "kinesis-lambda-dynamodb-pipeline-dotnet-cdk", |
| 28 | + "templateFile": "/src/KinesisLambdaDynamoDbCdk/KinesisLambdaDynamoDbCdkStack.cs" |
| 29 | + } |
| 30 | + }, |
| 31 | + "resources": { |
| 32 | + "bullets": [ |
| 33 | + { |
| 34 | + "text": "Amazon Kinesis Data Streams", |
| 35 | + "link": "https://docs.aws.amazon.com/streams/latest/dev/introduction.html" |
| 36 | + }, |
| 37 | + { |
| 38 | + "text": "AWS Lambda", |
| 39 | + "link": "https://docs.aws.amazon.com/lambda/latest/dg/welcome.html" |
| 40 | + }, |
| 41 | + { |
| 42 | + "text": "Amazon DynamoDB", |
| 43 | + "link": "https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Introduction.html" |
| 44 | + }, |
| 45 | + { |
| 46 | + "text": "AWS CDK", |
| 47 | + "link": "https://docs.aws.amazon.com/cdk/latest/guide/home.html" |
| 48 | + } |
| 49 | + ] |
| 50 | + }, |
| 51 | + "deploy": { |
| 52 | + "text": [ |
| 53 | + "cdk deploy" |
| 54 | + ] |
| 55 | + }, |
| 56 | + "testing": { |
| 57 | + "text": [ |
| 58 | + "See the GitHub repo for detailed testing instructions, including how to put records into the Kinesis stream and verify processing in DynamoDB tables." |
| 59 | + ] |
| 60 | + }, |
| 61 | + "cleanup": { |
| 62 | + "text": [ |
| 63 | + "Delete the stack: <code>cdk destroy</code>." |
| 64 | + ] |
| 65 | + }, |
| 66 | + "authors": [ |
| 67 | + { |
| 68 | + "name": "Tejas Vora", |
| 69 | + "image": "./TejasVora.jpg", |
| 70 | + "bio": "Tejas Vora is a Senior Solutions Architect with Amazon Web Services.", |
| 71 | + "linkedin": "tejas-vora-b4758a47" |
| 72 | + } |
| 73 | + ], |
| 74 | + "patternArch": { |
| 75 | + "icon1": { |
| 76 | + "x": 20, |
| 77 | + "y": 50, |
| 78 | + "service": "kinesis-datastreams", |
| 79 | + "label": "Amazon Kinesis" |
| 80 | + }, |
| 81 | + "icon2": { |
| 82 | + "x": 50, |
| 83 | + "y": 50, |
| 84 | + "service": "lambda", |
| 85 | + "label": "AWS Lambda" |
| 86 | + }, |
| 87 | + "icon3": { |
| 88 | + "x": 80, |
| 89 | + "y": 50, |
| 90 | + "service": "dynamodb", |
| 91 | + "label": "Amazon DynamoDB" |
| 92 | + }, |
| 93 | + "line1": { |
| 94 | + "from": "icon1", |
| 95 | + "to": "icon2", |
| 96 | + "label": "" |
| 97 | + }, |
| 98 | + |
| 99 | + "line2": { |
| 100 | + "from": "icon2", |
| 101 | + "to": "icon3", |
| 102 | + "label": "" |
| 103 | + } |
| 104 | + } |
| 105 | +} |
0 commit comments