diff --git a/typescript/opensearch/cwlogs_ingestion/.gitignore b/typescript/opensearch/cwlogs_ingestion/.gitignore new file mode 100644 index 0000000000..f60797b6a9 --- /dev/null +++ b/typescript/opensearch/cwlogs_ingestion/.gitignore @@ -0,0 +1,8 @@ +*.js +!jest.config.js +*.d.ts +node_modules + +# CDK asset staging directory +.cdk.staging +cdk.out diff --git a/typescript/opensearch/cwlogs_ingestion/.npmignore b/typescript/opensearch/cwlogs_ingestion/.npmignore new file mode 100644 index 0000000000..c1d6d45dcf --- /dev/null +++ b/typescript/opensearch/cwlogs_ingestion/.npmignore @@ -0,0 +1,6 @@ +*.ts +!*.d.ts + +# CDK asset staging directory +.cdk.staging +cdk.out diff --git a/typescript/opensearch/cwlogs_ingestion/README.md b/typescript/opensearch/cwlogs_ingestion/README.md new file mode 100644 index 0000000000..0c660dafeb --- /dev/null +++ b/typescript/opensearch/cwlogs_ingestion/README.md @@ -0,0 +1,70 @@ +## Amazon OpenSearch Ingestion + +## + +![Stability: Stable](https://img.shields.io/badge/stability-Stable-success.svg?style=for-the-badge) + +> **This is a stable example. It should successfully build out of the box** +> +> This example is built on Construct Libraries marked "Stable" and does not have any infrastructure prerequisites to build. + +--- + + + +## Overview + +[Amazon OpenSearch Ingestion](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/ingestion.html) is a fully managed, serverless data collector that delivers real-time log, metric, and trace data to [Amazon OpenSearch Service domains](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/what-is.html) and [Amazon OpenSearch Serverless collections](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless.html). + +In this example, we show how to use the [AWS Cloud Development Kit (CDK)](https://docs.aws.amazon.com/cdk/v2/guide/home.html) to set up an Amazon OpenSearch Ingestion Pipeline to ingest CloudWatch logs using a [CloudWatch subscription filter]() and write to Amazon OpenSearch Serverless collection in an [Amazon Virtual Private Cloud (VPC)](https://aws.amazon.com/vpc/). The pipeline is setup to receive log events on `/logs/ingest` path using the OpenSearch Ingestion pipeline API endpoint. + +![](docs/architecture.png) +_figure1.Architecture Diagram of loading CloudWatch logs into an OpenSearch Serverless collection using an OpenSearch Ingestion pipeline_ + +The CDK stack sets up roles and permissions to enable all of the services to communicate with one-another. It further provides access for the deploying user's IAM identity. Finally, the stack sets up a VPC, and creates a VPC endpoint for communication to OpenSearch Serverless (see below for configuration). + +### Configuration of the code + +To configure the solution for your account, visit [cwlogs_ingestion_stack](./cwlogs_ingestion/bin/cwlogs_ingestion_stack.ts). There are 2 stacks that are created here: +- `OpenSearchSetupStack` +- `CWLogsSubscriptionStack` + +You can modify the names of the resources in `OpenSearchSetupStack` stack. Below are default values: + +``` +STACK_NAMING_PREFIX = 'cw-to-os' + +STACK_RESOURCE_NAMING_PREFIX = 'OpenSearchSetup' + +COLLECTION_NAME = '${this.STACK_NAMING_PREFIX}-col' +PIPELINE_NAME = '${this.STACK_NAMING_PREFIX}-pipe' +NETWORK_POLICY_NAME = '${this.STACK_NAMING_PREFIX}-net-pol' +ENCRYPTION_POLICY_NAME = '${this.STACK_NAMING_PREFIX}-enc-pol' +DATA_ACCESS_POLICY_NAME = '${this.STACK_NAMING_PREFIX}-data-pol' +``` + +For, `CWLogsSubscriptionStack` stack, visit [cwlogs_subscription_stack](./cwlogs_ingestion/lib/cwlogs_subscription_stack.ts). This stack deploys the following resources: + +- `CWLogsSubscriptionLogEmitter` lambda to publish logs to CloudWatch which is invoked every 5 minutes triggered using an EventBridge timer. +- Lambda Subscription Filter for CloudWatch logs published by the `CWLogsSubscriptionLogEmitter` lambda resource + +To view data in your OpenSearch Serverless Collection navigate to the Amazon OpenSearch Service console. In the left navigation pane click the reveal triangle if it's not already open. Click **Collections**. Click **cw-to-os-col** (or find your **COLLECTION_NAME** if you changed it). Scroll down until you see the **Endpoint** section, and click the URL under **OpenSearch Dashboards URL**. This will launch OpenSearch Dashboards. + +In OpenSearch Dashboards, dismiss the initial splash screen. In the upper right, find the **Manage** link, and click it. Click **Index Patterns** in the left navigation pane. Click **Create index pattern**. and type `ddb-to-aoss-*` into the **Index pattern name** text box. Click **Next step**. Drop down the **Time field** menu, and select `@timestamp`. Click **Create index pattern**. + +## Build and Deploy + +The `cdk.json` file tells the CDK Toolkit how to execute your app. + +## Synthesize Cloudformation Template + +To see the Cloudformation template generated by the CDK, run `cdk synth`, then check the output file in the "cdk.out" directory. + +## Deploy + +- Run `cdk deploy OpenSearchSetupStack` to deploy OpenSearch Serverless collection, OpenSearch Ingestion Pipeline resources to your personal account. +- Run `cdk deploy CWLogsSubscriptionStack` to deploy CloudWatch logs subscription filter lambda resources to your personal account. + +## CDK Destroy + +If no longer want the stack to be running, you can destroy the stack by running `cdk destroy` diff --git a/typescript/opensearch/cwlogs_ingestion/bin/cwlogs_ingestion_stack.ts b/typescript/opensearch/cwlogs_ingestion/bin/cwlogs_ingestion_stack.ts new file mode 100644 index 0000000000..5c02d2b2c4 --- /dev/null +++ b/typescript/opensearch/cwlogs_ingestion/bin/cwlogs_ingestion_stack.ts @@ -0,0 +1,16 @@ +#!/usr/bin/env node +import "source-map-support/register"; +import { OpenSearchSetupStack } from "../lib/os_setup_stack"; +import { CWLogsSubscriptionStack } from "../lib/cwlogs_subscription_stack"; +import { App } from "aws-cdk-lib"; + +const app = new App(); +const opensearch_stack = new OpenSearchSetupStack( + app, + "OpenSearchSetupStack", + {}, +); + +new CWLogsSubscriptionStack(app, "CWLogsSubscriptionFilterStack", { + ingestionEndpointURL: opensearch_stack.ingestionEndPointURL, +}); \ No newline at end of file diff --git a/typescript/opensearch/cwlogs_ingestion/cdk.json b/typescript/opensearch/cwlogs_ingestion/cdk.json new file mode 100644 index 0000000000..ee1a49130f --- /dev/null +++ b/typescript/opensearch/cwlogs_ingestion/cdk.json @@ -0,0 +1,55 @@ +{ + "app": "npx ts-node --prefer-ts-exts bin/cwlogs_ingestion_stack.ts", + "watch": { + "include": ["**"], + "exclude": [ + "README.md", + "cdk*.json", + "**/*.d.ts", + "**/*.js", + "tsconfig.json", + "package*.json", + "yarn.lock", + "node_modules", + "test" + ] + }, + "context": { + "@aws-cdk/aws-lambda:recognizeLayerVersion": true, + "@aws-cdk/core:checkSecretUsage": true, + "@aws-cdk/core:target-partitions": ["aws", "aws-cn"], + "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, + "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, + "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true, + "@aws-cdk/aws-iam:minimizePolicies": true, + "@aws-cdk/core:validateSnapshotRemovalPolicy": true, + "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, + "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, + "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true, + "@aws-cdk/aws-apigateway:disableCloudWatchRole": true, + "@aws-cdk/core:enablePartitionLiterals": true, + "@aws-cdk/aws-events:eventsTargetQueueSameAccount": true, + "@aws-cdk/aws-iam:standardizedServicePrincipals": true, + "@aws-cdk/aws-ecs:disableExplicitDeploymentControllerForCircuitBreaker": true, + "@aws-cdk/aws-iam:importedRoleStackSafeDefaultPolicyName": true, + "@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true, + "@aws-cdk/aws-route53-patters:useCertificate": true, + "@aws-cdk/customresources:installLatestAwsSdkDefault": false, + "@aws-cdk/aws-rds:databaseProxyUniqueResourceName": true, + "@aws-cdk/aws-codedeploy:removeAlarmsFromDeploymentGroup": true, + "@aws-cdk/aws-apigateway:authorizerChangeDeploymentLogicalId": true, + "@aws-cdk/aws-ec2:launchTemplateDefaultUserData": true, + "@aws-cdk/aws-secretsmanager:useAttachedSecretResourcePolicyForSecretTargetAttachments": true, + "@aws-cdk/aws-redshift:columnId": true, + "@aws-cdk/aws-stepfunctions-tasks:enableEmrServicePolicyV2": true, + "@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": true, + "@aws-cdk/aws-apigateway:requestValidatorUniqueId": true, + "@aws-cdk/aws-kms:aliasNameRef": true, + "@aws-cdk/aws-autoscaling:generateLaunchTemplateInsteadOfLaunchConfig": true, + "@aws-cdk/core:includePrefixInUniqueNameGeneration": true, + "@aws-cdk/aws-efs:denyAnonymousAccess": true, + "@aws-cdk/aws-opensearchservice:enableOpensearchMultiAzWithStandby": true, + "@aws-cdk/aws-lambda-nodejs:useLatestRuntimeVersion": true, + "@aws-cdk/aws-efs:mountTargetOrderInsensitiveLogicalId": true + } +} diff --git a/typescript/opensearch/cwlogs_ingestion/docs/architecture.png b/typescript/opensearch/cwlogs_ingestion/docs/architecture.png new file mode 100644 index 0000000000..090fe68442 Binary files /dev/null and b/typescript/opensearch/cwlogs_ingestion/docs/architecture.png differ diff --git a/typescript/opensearch/cwlogs_ingestion/docs/architecture.xml b/typescript/opensearch/cwlogs_ingestion/docs/architecture.xml new file mode 100644 index 0000000000..6bf0520a0b --- /dev/null +++ b/typescript/opensearch/cwlogs_ingestion/docs/architecture.xml @@ -0,0 +1 @@ +7Z1Xe+I4F4B/TS6dx71cutKLMf1mHxe5gBu2wcCvX4mSACHM7EwmQzbkm/0SHUuyytE5r6VjeKLkaF3JzNRvJQ4In0jcWT9RyhNJkjyNw19IstlLCIIS9hIvC5yD7FVgBFtwEB4KesvAAflZxiJJwiJIz4V2EsfALs5kZpYl5Xk2NwnP75qaHngjMGwzfCsdBU7h76U8g7/KqyDw/OOdCfxwJTKPmQ+C3DedpDwRUeoTJWdJUuz/itYyCNHoHcflUC4G6wJdqTlDM1wemnVSQw/kyTKzgQJyOwvSIslgoewg3Of+54kSj60AWWCGwdYsgiTGViDL4e99rtUhi3kYsuxKxYd7GiAy4yKwFbMw5SQuzCAG2c/Uvi9dZEHsNYMCZGa4n7sCxMVZr9MsSUFWHNTGLwo03+ITqcF/MHsSJt7mOQf2MguKzbMZmdskfnbACl52k2Xs7FoAE05gepkZYasgX760DMpNkmQYgWExm3JcjLZJG7NoXoBJnHNZ3rRsgdu1RNu3udarvRnW/9QqqMWBF2NBnKdQT9FYanYSpUkMe57DBE+bPG4xLsawNInRJsFhAsMzGLBcx2IZl7aA/bFDk2/yAkRYhBYtnA8owWmGEhiBw2iSpzDaZRmMxwUcMwHLuYJDAYEWTgcF/nFdE45Xr+jm8dJB3a+r/suqy8714r8oPPFFFJ54KPy3U/jEmiFnReKhaUGPuesXzAI7gkYGtg3kxUF9oaLYQdjfpIc58AC8X2Bjtm9Cjxfuc724DHKRk+pMpMPSaLaa86RdoRmMflHEl4WWF5ujhwMOdHiHZJIVfuIlsRmqr1IpQ4MI0C1wmHrN00ySFAoJKJyBotgcvLe5LBIo8osoPFyFM55txofyu8QEJZ6ZY1JZn15UNofUvnOoge+ahSMTmJkHDqKYmK5xvlxsOsm4WRsGTMedH6rKHHeQwuWx780JKeQ5KNAYD09XMHUybhWQRAC2bbcmQqhRK3BRw3F6jvleinaTIEZzfYQi7mB2DvpM48J5FXsNOpQ6072TZvwn1/++VtyHfXzYmiu2RgrNeN6G5WvK1Vm9oeYf1QkPQT2WF9nSLpYZgBKW4CwAVyRmu3AMaRMnMJNjHIwngEOZNEcyFHnaiZuez6SFbrs/nOPzLPvHlXzQnxHB53hDnLFthiJszCFpAqMFm4KT4UCX6FAcx9kMTbLuh3rDH40rQzAcS+IYR1twXDkcwKYIDmYJlsuwFmtbtPXeyLw0oSzL55J6TjKkboQgCKijqA8kiUHLh+UbuDDXWAxXBFUgh/KzE3XTgNwbuNzsyXUn+7NjSuLo6U7Doc3XdgNq+9DuwdHcu/E/AApoDPescIMbPooT+PvhhFc0mJyRwXVO2KW6sLvQN6Ix3gkdM/d3jSH+E0mcrvBqqVXTcGhEPcIwgErFYvJP9y1wYLOM7XamFNF3BqWu1Bcrly/+BnActe+3uIB/cMGX5oIbSvvnuMAhXBtnWYBRPIsGEuAYT+E0xtrApUmXxs3DjtmDC/73XMDfJxf8aN3csOL3w9NfkiuIV4v+l8Di81jA76b+iJszvBtEFB+16z1nbN7P5gN/vvdAcReufN+VP773QNzRacQDMh6Q8YCMrwYZxJ2eE/5o4dzwD9+SMrpBimwUbCVeE1vw/3tJCN5Bj/SQFwvgRGUv+Y7w8a5Nepc88jkobP+IFcsCVS6/BBIgoQu7LsORz3YFKPg/DdUmITcegNdrMbT2KHsQhifZFYWiaRnK4WQkc3CR+QVK0I2gd4KjDZEGDU03yYNi76uspCiS6CSDGEJnAy8UCH0k85CyYVvQ/JxCD2r7gYgI8pg+dB5dN3f+CibcYI3aIaXI7YNMXe0c1z5T7pu74Y/WO7V6Nsucft6N/RGkUMPA+kco9SHwc7gLxZ5TDHGkmvI1QoM7PIX4J8EZNP1hWyc3/N+3YBraZaCNJ1zMpWwWWhQaMo2F4xhOUAyLm5YA3fEfdRl/2ifiFg2AafMYTgMGowHkNZ4VkI9mWNcSaM7i/qxPfGDpFSz9vxDdzX7ccIkff1BxsyWfQPkfxV2fiCydFMQGMDNIDiSuQCduJWbm5O9ACxymON/lxpyLvC/gQs2rFtvpAKVTnS+zAV7ZxMpfBRdepjVN+/+By+tk/HMyGX+PZDjmcj9GeCYY4fWH+ymwYS4iSH4HbN7TxQfYPMDmPjr4AJsvCjY3Hedno837du4bo80IdgnsJqcw3+GZXzn/IW7CzB0HoF4NLHn6pfOjG+F6nxlLcgAPDH/GcZY930Y5wMZvHjFhBHNWLX9eQeK6sLkX+v2xh0t3FPn/cHZfJbL1cbj0OFy6tCD3RgL/ixCWm/PxFkAesbl7lTz15181huYnGejGtuMXjKfdz9wDRx448sCRB478Eo5QXxNH/kqQ2B3Hupztf9V27x/tegEfwykCeeXXk78fHSUFx9JYelbmCBHv2qz3j5QO5yVBtPtEDGn3WzyetODXjl1+8ejncAfFQYxHifskqaXIeMvBUOr0SrxR8RIR/rSNga8OPPiXipKSKIsT+FtemOMqQJJapV0bzHFRrCYRlIs+lD2REipRU2AVEtGCf7LwmqgsUNGxXd+wg36JCqtTXGvruijmE4aHpYhKH4nFcdvo4TUxy2mb1ZFgGhi4o4pdHqloqaC5bpLxfFcJXKpauIV/dNboJrkuhUbeqcBr0pKp91RtANpZMaKKXFMHk52KSyYg/OliKPdETen1gtJoQ6EstWWRryd1DYxgNlX3GNWY+/W+q8KO64S9SCRdTiu4qpbwd6Ir5IZpLypm0BAj1KRKT220pWgu+nrQ98TN1rOsueVS3RYBKEKIGIqjFtS4GxksYNoOZUHdk3iKirvkiJUswiKbvTlOhK6qJ32py7Lzio2PBjxn8A0czMMm2AyGulT2VEGO65wqVV2y4dTs7cDEk2nPLcO5BTsoGcgq4nG7hcxiHsq4aGkhawVV02C2fWrWahQOGPKhsx174zlOWzOuDPBJZgRRF5pSiQHo00okFhc7DD8UlXq7ACMejloALLE9WY2cfJVTRSVqClzbtX1WbLRU0K01lWamx9O2u6J7cnssZA0S3kvUrHnVN6OS0tTWmiH4mSyI0RLeoLsm+7xKzeEylQZjPACTrbidCuN2HNU5srDkrNLvyHmrO3OU2mpRU6amZrL1pcqtu7TTqfvAT9vJnCBncEUNoNuQpoDdrnkPdbw5tk1B7Kx8VWd9zuklOFgV7HBTh9l68L91M1s2p/NWY5sHy6UrdniaspF6tLduQ3bwnq3Jq5jwlorOVmB+seYsssRqDV1j1IHOW9vmfXYWNBtRoz+HN2HIqkMMqmTLYddBsSQTbZJsvSCZOSLfzfh+UwuqDaluMNkEuSuxlfFRuZlwW06XhVVebcpMzWyrVtPue7FLZbhRa4ZiLRDx1obU1wNVbjvhUBgQ/oRWAmlgpait/mjZWUSCVycb03pvLpUjZVhfOFy3KEYjvjopCwlhoc/W6usQVLmhsU0mpSupyF7qc51tqOUYrn5Jt+VGsk59rcfMminT39SJVmUjqvo8mQwcv6JalXG/38gnMzVVY0JsNlr6uNUXKo2pMlsRYFxXhAbXabD8Qhd1oTWRnHBeW9cmxso38y1faYR0Nc3NLGn1qelEXA91B94WFOuaHhDLidodsPik11HKmbFoym15qa79fLwRe0N61BjInlcfU100cRNPpPMu4cbyJOqjkUzNTZQTndGMd5nxkKddPTXYXk7wKZ3j1KhCoRUvkcgqsVV6m8U1tel3h9O+xsc8IcAFH42Bmof6yO9krYzuzleRPF5OV3LQTnxtLUWLJW2KfqcJq/B9KtVVq2ttdFv1+vZULAR6zavJ2OeWabPXtsab2nS5NKNE4qL+IBKTQqRkJWrOamWvUec7eLxaw2ZXZUiXOLKVvDPoRBsxyNctpcGmm1q6mAh0paUk8lqIPJpbONOIRRY63bqF2YAarI2qta67oDMgVaNkHQeBv1bb6dDo5L2s2tyWkUU63KI9abmBWFkNcjAMK6I8LxsrbxtkPhGYdUl1u1F3ELq1bdFoSVFbSchmzfLJSY9Z1VoDl16FOa+ZCgPbqGltqJXNuGTtBlUvx9OSdy3L1Yv+3MnG1UaV7HRF+HCSK6UkEzkqoTDwSUOb083Sm9rjSTFAphtvuk2PWoak1qFhum63RaXVnSyWlUxfLiYM9JU4MHhbKmOyTmijeS43mgNa1Ydr0pYDXOTpVujR1HA72AhotY76puRzUtdJ5gbL9LxhKXhSDVnzWlSph0mDZxsTW6/UdUOquQ1u7jJgCdh5ezsgjTDLjHjebwarqjARiihX3VEz6MZ2uRBbMejZZh0X9drERxZqWtv0ACFVRjgYw6QzD7cjuYWTVbtQ1htqPpTTAWM681XOJJ4yCHQuWe88oT+olxFYzzjHXCSyZ5dBZMRFXV60yaxDT7IONysDaWHolWW0VpxGpd7ppGpHVC0xs8t1Zx1IxgTU6qBm6oFAGUYnmc7QpPRb4mwctVqjDBqiLafmRr0TzUqHDVvLbMuPCEj22kDsKPgQcd1AZYOJsOmF45VHacKA06oLx5ddry26Fcub5QZiaCmiaU2zElig39WDCC2eZR7g2YAYSTRpbcTBWibIadP1pVEMwtk4FXUwmAX92owWe2ozUUmiNU3Z/iTizM4yEzdrCBbSWKdVL9ku5tuK2xB06E16m+qsl7Q8hU4sOSzw3ibUN0QHqYVVRRZmRIwDuT9tIp9Wq1vGKECGik3NydxNRB9PRpNlrnTiPkjwRqcggQWGqpso0sJbL4Wk4w6BXKMW2txVK66S8LUFnZZ9QVfTitWzFzpfGttwWGXEfNyesp1piq4JlorXV9yKEyxfnqzVtONHMdXgu2p96mhqoDfqihfW3SXZHhuQj0Z8xR4gB0FVDE8YWpuJJLVwI5p467Y2cCeKarQndS/XaGLNtZMRWfpmCLU3ZxazWXtSJaW6WMmQaVE2Oa1NhmpJMM0wHTEj4MgDdTvoCV5Hl1StP88B02fUFvLvgwgZo7oqWSbdZ3qiNE8H0yRhXHfTG5slOZA0SU7wfMnXlC4hgmnmV7MVWPRxLhas+tgB3TmswZ14GbUxaScJLX2jEGqvFxnJKIpYr63OemIzElbLxO9l/txgunVvzLjN7iZgek209SBum2SL8FxECyUf5dFEXW7RUkkc1t1WjAbytbSBazyC9TybOHSrzkq9UakDpdjk1jz1Cs1SWj7TdUS5xw0ZezFyx31yuBrAhy4JNlHbWHYk2/V61LR3cGgMhp1eg5EntdqVsKWTMKWnvxYeTT2TZ0dwu7Ciw2HfaSgR8XwQngcTPbPch+2J3dg8+RYbYo94okc80SOe6BPiiX68n/LJEUafsG38BSOMhl35nd2tVWr/1EHY7U+z24UC77wGI8F/cCJl/ImBV2SUeiaZC8FlmjsXEG9TqI5zwWWaOxcQl9UTF/cnLht4IniTOqsev7g/ftJA+O+d2PB3YsBPUKb0oU4bqbmzpyW0+T+Iyka12q9as6vjMih7nycMzTQPrJdSGdLyPFgBqGL7yvH3gre9LFmmu+bX7J0rf3v1H6REl1HrKPyd5nmoOOdx7odu345PD4FboBrhWKClvksp1JUIe1GUOIm/jI7/WwzIkBeB5RT9Bv+IY/D5Kf0Rx6CwP3ce+p0+ePCBfw/8e+Dfn8G/F5T4ZLC7bda+Mdud8bgBMmh1Q5DDNcSGyIdacGmxHvoLus0Qlt516CoLgtCELG8fXtWCtnwV2BcnnO/Guv3cS3NXOfEqK17jxavM+JYbz7LtSO7KHS6F12TcWyHxNtsR/t4Kr8muke5laeJKaeKi9Puc+d47iD/3siG84u5+7vGNw+sfi3BYSXs8hWibXwPV67r9k6h42/Z87C7iOUCyz/jpz1ucvPZiIncR/P8bMHkjtvVbkCQAvOlQFIdxOAAYDR9pMBNncYymKciSgIGe8919jgeG3HdQ2l+H6J/fZnt165dO/JMp7BNi3b8ggr3DU78ScM7chKlPCDh/+qUY8xt6cRZj/v77n18xxpz5Rp7wKz6Q3+WbO48Y80eM+aUFuTff+cMY8z//Hv+XijEfBqB8uvkC/wrmwJyX6z+gAPaHFNDc3/zUc7951L58Io8Cx9lDAjr1MV+Pg863ZZRbj8WHb3w8FH56/ebAHz8uMx//uIzhz6RAU2fPzNjxYOf3XqinmItaLzz5J7xRz34jvHgc2TyObB57JX/oyObSO93H0Q17n+zzidggh8nSGZnFbo+nufuGrKvwYKN85T7fT9ADd5MefuuDC+E1jaJpnjy5pgTZYUcKRXdkaGf68rBBkpEZ/zaHDSfz9feCUYiLWBRWeD4gzecdH9xS0AfYPMDmPjr4AJsvCja3zcu9gcXNrryDAl/6c3iMpbWvYjde+NuIFC0IkZ/+uNOTv/5dmk8/fXryhb6+SriPr84m7ug7Mn/FLN3pN/T9XzzF//584cE5v3DE9yW++enmErzwoqc+80vTwREH8tSMX0JUTylIXafQA++M8lt0aJqR5ZhoPJbxIRxmnwVp1UmN78BFeCh+yhTvqsqf20rheBWn/9tWioIzMiTc77KVcpiov7iNItzBPsoNK3YPNPTnwzBZkiAogsAckyLRHgOPmRZJYDjuECyLWyQLPvibCR9P4N9m/+jmRH0CP3z+/sKJZ73iR78UWaD7Jklx+sSJNKyF1i0U/gs= \ No newline at end of file diff --git a/typescript/opensearch/cwlogs_ingestion/lib/cwlogs_subscription_stack.ts b/typescript/opensearch/cwlogs_ingestion/lib/cwlogs_subscription_stack.ts new file mode 100644 index 0000000000..77e89eddf2 --- /dev/null +++ b/typescript/opensearch/cwlogs_ingestion/lib/cwlogs_subscription_stack.ts @@ -0,0 +1,99 @@ +import { PythonLayerVersion } from '@aws-cdk/aws-lambda-python-alpha'; +import { Duration, Stack, StackProps } from 'aws-cdk-lib'; +import { Rule, Schedule } from 'aws-cdk-lib/aws-events'; +import { LambdaFunction } from 'aws-cdk-lib/aws-events-targets'; +import { Effect, PolicyStatement } from 'aws-cdk-lib/aws-iam'; +import { Code, Runtime, Function, Alias } from 'aws-cdk-lib/aws-lambda'; +import { FilterPattern, LogGroup, RetentionDays, SubscriptionFilter } from 'aws-cdk-lib/aws-logs'; +import { LambdaDestination } from 'aws-cdk-lib/aws-logs-destinations'; +import { Construct } from 'constructs'; +import path = require('path'); + +export interface CWLogsSubscriptionStackProps extends StackProps { + ingestionEndpointURL: string +} + +export class CWLogsSubscriptionStack extends Stack { + + private readonly STACK_NAMING_PREFIX: string = 'cwlogs-subscription'; + + constructor(scope: Construct, id: string, props: CWLogsSubscriptionStackProps) { + super(scope, id, props); + + ///////////////////////////////////////////////////////////////////////////////// + // + // Create the Log Emitter Lambda resources + // + ///////////////////////////////////////////////////////////////////////////////// + const logGroup = new LogGroup(this, `EventBridgeTriggeredLambdaLogGroup`, { + retention: RetentionDays.ONE_WEEK, + }); + + // Lambda Function to publish message + const lambdaFn = new Function(this, 'EventBridgeTriggeredLambdaFunction', { + code: Code.fromAsset(path.join(__dirname, '../resources/lambda/log_emitter')), + handler: 'handler.log_emitter', + timeout: Duration.seconds(300), + runtime: Runtime.PYTHON_3_12, + logGroup: logGroup + }); + + // Run the eventbridge every 5 minute interval to generate logs + const rule = new Rule(this, 'Rule', { + schedule: Schedule.rate(Duration.minutes(5)) + }); + + // Add the lambda function as a target to the eventbridge + rule.addTarget(new LambdaFunction(lambdaFn)); + + + ///////////////////////////////////////////////////////////////////////////////// + // + // Create the CloudWatch Log group subscription filter resources + // + ///////////////////////////////////////////////////////////////////////////////// + + const lambdaLayer = new PythonLayerVersion(this, `${this.STACK_NAMING_PREFIX}LambdaLayer`, { + entry: path.join(__dirname, "../resources/lambda/cw_subscription_filter/layers"), + compatibleRuntimes: [ + Runtime.PYTHON_3_12, + Runtime.PYTHON_3_11, + ], + description: "A layer that contains the required modules", + license: "MIT License", + }); + + const lambdaFunction = new Function(this, `${this.STACK_NAMING_PREFIX}LambdaFunction`, { + runtime: Runtime.PYTHON_3_12, + code: Code.fromAsset(path.join(__dirname, '../resources/lambda/cw_subscription_filter')), + handler: 'handler.cw_subscription_handler', + layers: [lambdaLayer], + environment: { + OSI_INGESTION_ENDPOINT: props.ingestionEndpointURL, + }, + } + ); + + new Alias(this, `${this.STACK_NAMING_PREFIX}LambdaFunctionAlias`, { + aliasName: 'live', + version: lambdaFunction.currentVersion, + }); + + lambdaFunction.addToRolePolicy( + new PolicyStatement({ + effect: Effect.ALLOW, + resources: ['*'], + actions: ['osis:ingest'], + }), + ); + + // Create a Lambda Subscription Filter on the specific log group created above + const subscriptionFilter = new SubscriptionFilter(this, `${this.STACK_NAMING_PREFIX}LogSubscription`, { + logGroup: logGroup, + destination: new LambdaDestination(lambdaFunction), + filterPattern: FilterPattern.allEvents(), + }); + subscriptionFilter.node.addDependency(lambdaFunction); + } + +} diff --git a/typescript/opensearch/cwlogs_ingestion/lib/os_setup_stack.ts b/typescript/opensearch/cwlogs_ingestion/lib/os_setup_stack.ts new file mode 100644 index 0000000000..bae2882c4a --- /dev/null +++ b/typescript/opensearch/cwlogs_ingestion/lib/os_setup_stack.ts @@ -0,0 +1,244 @@ +import { Fn, Stack, StackProps } from 'aws-cdk-lib'; +import { Peer, Port, SecurityGroup, Vpc } from 'aws-cdk-lib/aws-ec2'; +import { Effect, Policy, PolicyDocument, PolicyStatement, Role, AccountPrincipal, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; +import { CfnAccessPolicy, CfnCollection, CfnSecurityPolicy, CfnVpcEndpoint } from 'aws-cdk-lib/aws-opensearchserverless'; +import { CfnPipeline } from 'aws-cdk-lib/aws-osis'; +import { Construct } from 'constructs'; +import { readFileSync } from "fs"; + +export class OpenSearchSetupStack extends Stack { + + private readonly STACK_NAMING_PREFIX: string = 'cw-to-os'; + private readonly STACK_RESOURCE_NAMING_PREFIX: string = 'OpenSearchSetup'; + private readonly COLLECTION_NAME: string = `${this.STACK_NAMING_PREFIX}-col`; + private readonly DATA_ACCESS_POLICY_NAME: string = `${this.STACK_NAMING_PREFIX}-data-pol`; + private readonly NETWORK_POLICY_NAME: string = `${this.STACK_NAMING_PREFIX}-net-pol`; + private readonly ENCRYPTION_POLICY_NAME: string = `${this.STACK_NAMING_PREFIX}-enc-pol`; + private readonly VPC_ENDPOINT_NAME: string = `${this.STACK_NAMING_PREFIX}-vpc`; + private readonly PIPELINE_NAME: string = `${this.STACK_NAMING_PREFIX}-pipe`; + + public readonly ingestionEndPointURL: string; + + constructor(scope: Construct, id: string, props?: StackProps) { + super(scope, id, props); + + // Create VPC + const vpc = new Vpc(this, `${this.STACK_RESOURCE_NAMING_PREFIX}-vpc`); + + // Create Security Group + const securityGroup = new SecurityGroup(this, `${this.STACK_RESOURCE_NAMING_PREFIX}-security-group`, { + description: 'Security group for OpenSearch', + vpc: vpc, + allowAllOutbound: true, + }); + + securityGroup.addIngressRule(Peer.anyIpv4(), Port.tcp(443)); + securityGroup.connections.allowFrom( + securityGroup, + Port.allTraffic(), + 'Allow ingress from the same SecurityGroup', + ); + + // Create VPC Endpoint + const vpcEndpoint = new CfnVpcEndpoint(this, `${this.STACK_RESOURCE_NAMING_PREFIX}VpcEndpoint`, { + name: this.VPC_ENDPOINT_NAME, + vpcId: vpc.vpcId, + subnetIds: vpc.privateSubnets.map((subnet) => subnet.subnetId), + securityGroupIds: [securityGroup.securityGroupId], + }); + + // Create OpenSearch Serverless network security policy + const cfnNetworkAccessPolicy = new CfnSecurityPolicy(this, `${this.STACK_RESOURCE_NAMING_PREFIX}NetworkPolicy`, { + name: this.NETWORK_POLICY_NAME, + type: 'network', + policy: JSON.stringify([ + { + AllowFromPublic: false, + Rules: [ + { + ResourceType: 'collection', + Resource: [`collection/${this.COLLECTION_NAME}`], + }, + ], + SourceVPCEs: [vpcEndpoint.attrId], + }, + { + AllowFromPublic: true, + Rules: [ + { + ResourceType: 'dashboard', + Resource: [`collection/${this.COLLECTION_NAME}`], + }, + ], + }, + ]), + }); + + // Create OpenSearch Serverless encryption policy + const cfnEncryptionPolicy = new CfnSecurityPolicy(this, `${this.STACK_RESOURCE_NAMING_PREFIX}EncryptionPolicy`, { + name: this.ENCRYPTION_POLICY_NAME, + type: 'encryption', + policy: JSON.stringify({ + Rules: [ + { + ResourceType: 'collection', + Resource: [`collection/${this.COLLECTION_NAME}`], + } + ], + AWSOwnedKey: true + }) + }); + + // Create OpenSearch Serverless collection + const cfnCollection = new CfnCollection( + this, + `${this.COLLECTION_NAME}Collection`, + { + name: this.COLLECTION_NAME, + description: 'OpenSearch serverless collection to be used for search from CDK', + type: 'SEARCH', + }, + ); + + cfnCollection.addDependency(cfnEncryptionPolicy); + cfnCollection.addDependency(cfnNetworkAccessPolicy); + + // Create IAM role for OpenSearch Ingestion pipeline + const pipelineRole = new Role(this, `${this.STACK_RESOURCE_NAMING_PREFIX}PipelineRole`, { + roleName: `${this.STACK_RESOURCE_NAMING_PREFIX}PipelineRole`, + assumedBy: new ServicePrincipal('osis-pipelines.amazonaws.com'), + inlinePolicies: { + 'OSISPipelineRolePolicy': this.pipelinePolicies(cfnCollection.attrArn) + } + }); + + // Create OpenSearch Ingestion pipeline + const cfnPipeline = new CfnPipeline(this, `${this.STACK_RESOURCE_NAMING_PREFIX}Pipeline`, { + maxUnits: 4, + minUnits: 2, + bufferOptions: { + persistentBufferEnabled: true + }, + pipelineConfigurationBody: this.getPipelineConfiguration(pipelineRole.roleArn, cfnCollection.attrCollectionEndpoint), + pipelineName: this.PIPELINE_NAME, + }); + + this.ingestionEndPointURL = Fn.select(0, cfnPipeline.attrIngestEndpointUrls); + + cfnPipeline.addDependency(cfnCollection); + + // Create a dashboard access role + const dashboardAccessRole = new Role(this, `${this.STACK_RESOURCE_NAMING_PREFIX}DashboardAccessRole`, { + assumedBy: new AccountPrincipal(this.account) , + }); + + dashboardAccessRole.attachInlinePolicy( + new Policy(this, `${this.STACK_RESOURCE_NAMING_PREFIX}DashboardAccessPolicy`, { + statements: [ + new PolicyStatement({ + effect: Effect.ALLOW, + resources: ['*'], + actions: ['aoss:*'], + }), + ], + }), + ); + + // Create OpenSearch Serverless data access policy + const data_access_policy_arns: string[] = [pipelineRole.roleArn, dashboardAccessRole.roleArn]; + + const cfnDataAccessPolicy = new CfnAccessPolicy(this, `${this.STACK_NAMING_PREFIX}AccessPolicy`, { + name: this.DATA_ACCESS_POLICY_NAME, + type: 'data', + policy: JSON.stringify([ + { + Rules: [ + { + ResourceType: 'index', + Resource: [`index/${this.COLLECTION_NAME}/*`], + Permission: [ + 'aoss:CreateIndex', + 'aoss:DescribeIndex', + 'aoss:ReadDocument', + 'aoss:WriteDocument', + 'aoss:UpdateIndex', + 'aoss:DeleteIndex', + ], + }, + { + ResourceType: 'collection', + Resource: [`collection/${this.COLLECTION_NAME}`], + Permission: [ + 'aoss:CreateCollectionItems', + 'aoss:DeleteCollectionItems', + 'aoss:UpdateCollectionItems', + 'aoss:DescribeCollectionItems', + ], + }, + ], + Principal: data_access_policy_arns, + }, + ]), + }); + + cfnDataAccessPolicy.addDependency(cfnCollection); + cfnDataAccessPolicy.addDependency(cfnPipeline); + } + + pipelinePolicies(collectionArn: string) { + const policyDocument = new PolicyDocument(); + policyDocument.addStatements( + new PolicyStatement({ + 'effect': Effect.ALLOW, + "resources": ["*"], + "actions": [ + "aoss:BatchGetCollection" + ] + })); + policyDocument.addStatements( + new PolicyStatement({ + 'effect': Effect.ALLOW, + "resources": [collectionArn], + "actions": [ + "aoss:APIAccessAll" + ] + })); + policyDocument.addStatements( + new PolicyStatement({ + 'effect': Effect.ALLOW, + "resources": [`arn:aws:aoss:*:${this.account}:dashboards/default`], + "actions": [ + "aoss:DashboardsAccessAll" + ] + })); + policyDocument.addStatements( + new PolicyStatement({ + 'effect': Effect.ALLOW, + "resources": ["*"], + "actions": [ + "aoss:CreateSecurityPolicy", + "aoss:GetSecurityPolicy", + "aoss:UpdateSecurityPolicy" + ], + "conditions": { + "StringEquals": { + "aoss:collection": this.COLLECTION_NAME + } + } + })); + return policyDocument; + } + + getPipelineConfiguration(roleArn: string, collectionEndPoint: string) { + let pipelineConfigurationTemplate = readFileSync('resources/pipeline/configuration.yaml').toString() + + const formattedPipelineConfiguration = + pipelineConfigurationTemplate + .replace('', collectionEndPoint) + .replace('', roleArn) + .replace('', this.region) + .replace('', this.NETWORK_POLICY_NAME); + + return formattedPipelineConfiguration; + } +} diff --git a/typescript/opensearch/cwlogs_ingestion/package.json b/typescript/opensearch/cwlogs_ingestion/package.json new file mode 100644 index 0000000000..cba042315d --- /dev/null +++ b/typescript/opensearch/cwlogs_ingestion/package.json @@ -0,0 +1,32 @@ +{ + "name": "cwlogs_ingestion", + "version": "0.1.0", + "bin": { + "cwlogs_ingestion": "bin/cwlogs_ingestion.js" + }, + "scripts": { + "clean": "rm -rf build dist cdk.out node_modules && find . -name \"*.js\" -type f -delete && find . -name \"*.d.ts\" -type f -delete", + "build": "tsc", + "watch": "tsc -w", + "cdk": "cdk" + }, + "devDependencies": { + "@types/jest": "^29.5.4", + "@types/node": "20.5.9", + "aws-cdk": "^2.165.0", + "aws-cdk-lib": "^2.165.0", + "constructs": "^10.2.43", + "globals": "^15.6.0", + "jest": "^29.6.4", + "ts-jest": "^29.1.1", + "ts-node": "^10.9.1", + "typescript": "~5.2.2" + }, + "dependencies": { + "@aws-cdk/aws-lambda-python-alpha": "2.165.0-alpha.0", + "aws-cdk": "^2.165.0", + "aws-cdk-lib": "^2.165.0", + "constructs": "^10.0.0", + "source-map-support": "^0.5.21" + } +} diff --git a/typescript/opensearch/cwlogs_ingestion/resources/lambda/cw_subscription_filter/handler.py b/typescript/opensearch/cwlogs_ingestion/resources/lambda/cw_subscription_filter/handler.py new file mode 100644 index 0000000000..b0a4090e33 --- /dev/null +++ b/typescript/opensearch/cwlogs_ingestion/resources/lambda/cw_subscription_filter/handler.py @@ -0,0 +1,57 @@ +import base64 +import gzip +import logging +import json +import jmespath +import requests +import os +from datetime import datetime +from requests_auth_aws_sigv4 import AWSSigV4 + +def cw_subscription_handler(event, context): + + """Extract the data from the event""" + data = jmespath.search("awslogs.data", event) + + """Decompress the logs""" + cwLogs = decompress_json_data(data) + + """Construct the payload to send to OpenSearch Ingestion""" + payload = prepare_payload(cwLogs) + + """Ingest the set of events to the pipeline""" + response = ingestData(payload) + + return { + 'statusCode': 200, + 'execute-api': { + 'status_code': response.status_code, + 'response': response.text + } + } +def decompress_json_data(data): + compressed_data = base64.b64decode(data) + uncompressed_data = gzip.decompress(compressed_data) + return json.loads(uncompressed_data) + +def prepare_payload(cwLogs): + payload = [] + logEvents = cwLogs['logEvents'] + for logEvent in logEvents: + request = {} + request['@id'] = logEvent['id'] + request['@timestamp'] = str(datetime.now().year) + '0' + str(datetime.now().month) + '0' + str(datetime.now().day) + request['@message'] = logEvent['message'] + request['@owner'] = cwLogs['owner'] + request['@log_group'] = cwLogs['logGroup'] + request['@log_stream'] = cwLogs['logStream'] + + payload.append(request) + return payload + +def ingestData(payload): + ingestionEndpoint = os.environ["OSI_INGESTION_ENDPOINT"] + endpoint = 'https://' + ingestionEndpoint + response = requests.request('POST', f'{endpoint}/logs/ingest', data=json.dumps(payload), auth=AWSSigV4('osis')) + print('Response received: ' + response.text) + return response \ No newline at end of file diff --git a/typescript/opensearch/cwlogs_ingestion/resources/lambda/cw_subscription_filter/layers/requirements.txt b/typescript/opensearch/cwlogs_ingestion/resources/lambda/cw_subscription_filter/layers/requirements.txt new file mode 100644 index 0000000000..7bfeb86a6c --- /dev/null +++ b/typescript/opensearch/cwlogs_ingestion/resources/lambda/cw_subscription_filter/layers/requirements.txt @@ -0,0 +1,3 @@ +boto3 +requests +requests_auth_aws_sigv4 \ No newline at end of file diff --git a/typescript/opensearch/cwlogs_ingestion/resources/lambda/log_emitter/handler.py b/typescript/opensearch/cwlogs_ingestion/resources/lambda/log_emitter/handler.py new file mode 100644 index 0000000000..d66d37bbba --- /dev/null +++ b/typescript/opensearch/cwlogs_ingestion/resources/lambda/log_emitter/handler.py @@ -0,0 +1,15 @@ +from datetime import datetime +import json +from random import randrange + +def log_emitter(event, context): + source = {} + id = str(randrange(10000)) + source['id'] = id + source['timestamp'] = str(datetime.now()) + source['message'] = 'Hello world' + source['owner'] = 'aws-osi' + + print(json.dumps(source)) + + diff --git a/typescript/opensearch/cwlogs_ingestion/resources/pipeline/configuration.yaml b/typescript/opensearch/cwlogs_ingestion/resources/pipeline/configuration.yaml new file mode 100644 index 0000000000..286eaf010e --- /dev/null +++ b/typescript/opensearch/cwlogs_ingestion/resources/pipeline/configuration.yaml @@ -0,0 +1,15 @@ +version: "2" +cwlogs-ingestion-http-pipeline: + source: + http: + path: /logs/ingest + sink: + - opensearch: + hosts: [""] + aws: + sts_role_arn: "" + region: "" + serverless: true + serverless_options: + network_policy_name: "" + index: "cwl-%{yyyy-MM-dd}" diff --git a/typescript/opensearch/cwlogs_ingestion/tsconfig.json b/typescript/opensearch/cwlogs_ingestion/tsconfig.json new file mode 100644 index 0000000000..464ed774ba --- /dev/null +++ b/typescript/opensearch/cwlogs_ingestion/tsconfig.json @@ -0,0 +1,23 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "commonjs", + "lib": ["es2020", "dom"], + "declaration": true, + "strict": true, + "noImplicitAny": true, + "strictNullChecks": true, + "noImplicitThis": true, + "alwaysStrict": true, + "noUnusedLocals": false, + "noUnusedParameters": false, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": false, + "inlineSourceMap": true, + "inlineSources": true, + "experimentalDecorators": true, + "strictPropertyInitialization": false, + "typeRoots": ["./node_modules/@types"] + }, + "exclude": ["node_modules", "cdk.out"] +}