Add an example for ingesting CloudWatch logs into OpenSearch using OpenSearch Ingestion pipeline.

typescript/opensearch/cwlogs_ingestion/.gitignore

@@ -0,0 +1,8 @@
+*.js
+!jest.config.js
+*.d.ts
+node_modules
+
+# CDK asset staging directory
+.cdk.staging
+cdk.out

typescript/opensearch/cwlogs_ingestion/.npmignore

@@ -0,0 +1,6 @@
+*.ts
+!*.d.ts
+
+# CDK asset staging directory
+.cdk.staging
+cdk.out

typescript/opensearch/cwlogs_ingestion/README.md

@@ -0,0 +1,70 @@
+## Amazon OpenSearch Ingestion
+
+## <!--BEGIN STABILITY BANNER-->
+
+![Stability: Stable](https://img.shields.io/badge/stability-Stable-success.svg?style=for-the-badge)
+
+> **This is a stable example. It should successfully build out of the box**
+>
+> This example is built on Construct Libraries marked "Stable" and does not have any infrastructure prerequisites to build.
+
+---
+
+<!--END STABILITY BANNER-->
+
+## Overview
+
+[Amazon OpenSearch Ingestion](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/ingestion.html) is a fully managed, serverless data collector that delivers real-time log, metric, and trace data to [Amazon OpenSearch Service domains](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/what-is.html) and [Amazon OpenSearch Serverless collections](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless.html).
+
+In this example, we show how to use the [AWS Cloud Development Kit (CDK)](https://docs.aws.amazon.com/cdk/v2/guide/home.html) to set up an Amazon OpenSearch Ingestion Pipeline to ingest CloudWatch logs using a [CloudWatch subscription filter]() and write to Amazon OpenSearch Serverless collection in an [Amazon Virtual Private Cloud (VPC)](https://aws.amazon.com/vpc/). The pipeline is setup to receive log events on `/logs/ingest` path using the OpenSearch Ingestion pipeline API endpoint. 
+
+![](docs/architecture.png)
+_figure1.Architecture Diagram of loading CloudWatch logs into an OpenSearch Serverless collection using an OpenSearch Ingestion pipeline_
+
+The CDK stack sets up roles and permissions to enable all of the services to communicate with one-another. It further provides access for the deploying user's IAM identity. Finally, the stack sets up a VPC, and creates a VPC endpoint for communication to OpenSearch Serverless (see below for configuration).
+
+### Configuration of the code
+
+To configure the solution for your account, visit [cwlogs_ingestion_stack](./cwlogs_ingestion/bin/cwlogs_ingestion_stack.ts). There are 2 stacks that are created here:
+- `OpenSearchSetupStack`
+- `CWLogsSubscriptionStack`
+
+You can modify the names of the resources in `OpenSearchSetupStack` stack. Below are default values:
+
+```
+STACK_NAMING_PREFIX = 'cw-to-os'
+
+STACK_RESOURCE_NAMING_PREFIX = 'OpenSearchSetup'
+
+COLLECTION_NAME = '${this.STACK_NAMING_PREFIX}-col'
+PIPELINE_NAME = '${this.STACK_NAMING_PREFIX}-pipe'
+NETWORK_POLICY_NAME = '${this.STACK_NAMING_PREFIX}-net-pol'
+ENCRYPTION_POLICY_NAME = '${this.STACK_NAMING_PREFIX}-enc-pol'
+DATA_ACCESS_POLICY_NAME = '${this.STACK_NAMING_PREFIX}-data-pol'
+```
+
+For, `CWLogsSubscriptionStack` stack, visit [cwlogs_subscription_stack](./cwlogs_ingestion/lib/cwlogs_subscription_stack.ts). This stack deploys the following resources:
+
+- `CWLogsSubscriptionLogEmitter` lambda to publish logs to CloudWatch which is invoked every 5 minutes triggered using an EventBridge timer. 
+- Lambda Subscription Filter for CloudWatch logs published by the `CWLogsSubscriptionLogEmitter` lambda resource
+
+To view data in your OpenSearch Serverless Collection navigate to the Amazon OpenSearch Service console. In the left navigation pane click the reveal triangle if it's not already open. Click **Collections**. Click **cw-to-os-col** (or find your **COLLECTION_NAME** if you changed it). Scroll down until you see the **Endpoint** section, and click the URL under **OpenSearch Dashboards URL**. This will launch OpenSearch Dashboards.
+
+In OpenSearch Dashboards, dismiss the initial splash screen. In the upper right, find the **Manage** link, and click it. Click **Index Patterns** in the left navigation pane. Click **Create index pattern**. and type `ddb-to-aoss-*` into the **Index pattern name** text box. Click **Next step**. Drop down the **Time field** menu, and select `@timestamp`. Click **Create index pattern**.
+
+## Build and Deploy
+
+The `cdk.json` file tells the CDK Toolkit how to execute your app.
+
+## Synthesize Cloudformation Template
+
+To see the Cloudformation template generated by the CDK, run `cdk synth`, then check the output file in the "cdk.out" directory.
+
+## Deploy
+
+- Run `cdk deploy OpenSearchSetupStack` to deploy OpenSearch Serverless collection, OpenSearch Ingestion Pipeline resources to your personal account.
+- Run `cdk deploy CWLogsSubscriptionStack` to deploy CloudWatch logs subscription filter lambda resources to your personal account.
+
+## CDK Destroy
+
+If no longer want the stack to be running, you can destroy the stack by running `cdk destroy`

typescript/opensearch/cwlogs_ingestion/bin/cwlogs_ingestion_stack.ts

@@ -0,0 +1,16 @@
+#!/usr/bin/env node
+import "source-map-support/register";
+import { OpenSearchSetupStack } from "../lib/os_setup_stack";
+import { CWLogsSubscriptionStack } from "../lib/cwlogs_subscription_stack";
+import { App } from "aws-cdk-lib";
+
+const app = new App();
+const opensearch_stack = new OpenSearchSetupStack(
+  app,
+  "OpenSearchSetupStack",
+  {},
+);
+
+new CWLogsSubscriptionStack(app, "CWLogsSubscriptionFilterStack", {
+  ingestionEndpointURL: opensearch_stack.ingestionEndPointURL,
+});

typescript/opensearch/cwlogs_ingestion/cdk.json

@@ -0,0 +1,55 @@
+{
+  "app": "npx ts-node --prefer-ts-exts bin/cwlogs_ingestion_stack.ts",
+  "watch": {
+    "include": ["**"],
+    "exclude": [
+      "README.md",
+      "cdk*.json",
+      "**/*.d.ts",
+      "**/*.js",
+      "tsconfig.json",
+      "package*.json",
+      "yarn.lock",
+      "node_modules",
+      "test"
+    ]
+  },
+  "context": {
+    "@aws-cdk/aws-lambda:recognizeLayerVersion": true,
+    "@aws-cdk/core:checkSecretUsage": true,
+    "@aws-cdk/core:target-partitions": ["aws", "aws-cn"],
+    "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true,
+    "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true,
+    "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true,
+    "@aws-cdk/aws-iam:minimizePolicies": true,
+    "@aws-cdk/core:validateSnapshotRemovalPolicy": true,
+    "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true,
+    "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true,
+    "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true,
+    "@aws-cdk/aws-apigateway:disableCloudWatchRole": true,
+    "@aws-cdk/core:enablePartitionLiterals": true,
+    "@aws-cdk/aws-events:eventsTargetQueueSameAccount": true,
+    "@aws-cdk/aws-iam:standardizedServicePrincipals": true,
+    "@aws-cdk/aws-ecs:disableExplicitDeploymentControllerForCircuitBreaker": true,
+    "@aws-cdk/aws-iam:importedRoleStackSafeDefaultPolicyName": true,
+    "@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true,
+    "@aws-cdk/aws-route53-patters:useCertificate": true,
+    "@aws-cdk/customresources:installLatestAwsSdkDefault": false,
+    "@aws-cdk/aws-rds:databaseProxyUniqueResourceName": true,
+    "@aws-cdk/aws-codedeploy:removeAlarmsFromDeploymentGroup": true,
+    "@aws-cdk/aws-apigateway:authorizerChangeDeploymentLogicalId": true,
+    "@aws-cdk/aws-ec2:launchTemplateDefaultUserData": true,
+    "@aws-cdk/aws-secretsmanager:useAttachedSecretResourcePolicyForSecretTargetAttachments": true,
+    "@aws-cdk/aws-redshift:columnId": true,
+    "@aws-cdk/aws-stepfunctions-tasks:enableEmrServicePolicyV2": true,
+    "@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": true,
+    "@aws-cdk/aws-apigateway:requestValidatorUniqueId": true,
+    "@aws-cdk/aws-kms:aliasNameRef": true,
+    "@aws-cdk/aws-autoscaling:generateLaunchTemplateInsteadOfLaunchConfig": true,
+    "@aws-cdk/core:includePrefixInUniqueNameGeneration": true,
+    "@aws-cdk/aws-efs:denyAnonymousAccess": true,
+    "@aws-cdk/aws-opensearchservice:enableOpensearchMultiAzWithStandby": true,
+    "@aws-cdk/aws-lambda-nodejs:useLatestRuntimeVersion": true,
+    "@aws-cdk/aws-efs:mountTargetOrderInsensitiveLogicalId": true
+  }
+}

typescript/opensearch/cwlogs_ingestion/docs/architecture.xml

@@ -0,0 +1 @@
+<mxfile modified="2024-07-01T16:15:37.677Z" host="design-inspector.a2z.com" agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36" etag="wU0Vv1WqCyz69K9MQpfh" version="10.1.8" type="device"><diagram id="a49PNTVk0krr_fBheTj1i" name="Page-1">7Z1Xe+I4F4B/TS6dx71cutKLMf1mHxe5gBu2wcCvX4mSACHM7EwmQzbkm/0SHUuyytE5r6VjeKLkaF3JzNRvJQ4In0jcWT9RyhNJkjyNw19IstlLCIIS9hIvC5yD7FVgBFtwEB4KesvAAflZxiJJwiJIz4V2EsfALs5kZpYl5Xk2NwnP75qaHngjMGwzfCsdBU7h76U8g7/KqyDw/OOdCfxwJTKPmQ+C3DedpDwRUeoTJWdJUuz/itYyCNHoHcflUC4G6wJdqTlDM1wemnVSQw/kyTKzgQJyOwvSIslgoewg3Of+54kSj60AWWCGwdYsgiTGViDL4e99rtUhi3kYsuxKxYd7GiAy4yKwFbMw5SQuzCAG2c/Uvi9dZEHsNYMCZGa4n7sCxMVZr9MsSUFWHNTGLwo03+ITqcF/MHsSJt7mOQf2MguKzbMZmdskfnbACl52k2Xs7FoAE05gepkZYasgX760DMpNkmQYgWExm3JcjLZJG7NoXoBJnHNZ3rRsgdu1RNu3udarvRnW/9QqqMWBF2NBnKdQT9FYanYSpUkMe57DBE+bPG4xLsawNInRJsFhAsMzGLBcx2IZl7aA/bFDk2/yAkRYhBYtnA8owWmGEhiBw2iSpzDaZRmMxwUcMwHLuYJDAYEWTgcF/nFdE45Xr+jm8dJB3a+r/suqy8714r8oPPFFFJ54KPy3U/jEmiFnReKhaUGPuesXzAI7gkYGtg3kxUF9oaLYQdjfpIc58AC8X2Bjtm9Cjxfuc724DHKRk+pMpMPSaLaa86RdoRmMflHEl4WWF5ujhwMOdHiHZJIVfuIlsRmqr1IpQ4MI0C1wmHrN00ySFAoJKJyBotgcvLe5LBIo8osoPFyFM55txofyu8QEJZ6ZY1JZn15UNofUvnOoge+ahSMTmJkHDqKYmK5xvlxsOsm4WRsGTMedH6rKHHeQwuWx780JKeQ5KNAYD09XMHUybhWQRAC2bbcmQqhRK3BRw3F6jvleinaTIEZzfYQi7mB2DvpM48J5FXsNOpQ6072TZvwn1/++VtyHfXzYmiu2RgrNeN6G5WvK1Vm9oeYf1QkPQT2WF9nSLpYZgBKW4CwAVyRmu3AMaRMnMJNjHIwngEOZNEcyFHnaiZuez6SFbrs/nOPzLPvHlXzQnxHB53hDnLFthiJszCFpAqMFm4KT4UCX6FAcx9kMTbLuh3rDH40rQzAcS+IYR1twXDkcwKYIDmYJlsuwFmtbtPXeyLw0oSzL55J6TjKkboQgCKijqA8kiUHLh+UbuDDXWAxXBFUgh/KzE3XTgNwbuNzsyXUn+7NjSuLo6U7Doc3XdgNq+9DuwdHcu/E/AApoDPescIMbPooT+PvhhFc0mJyRwXVO2KW6sLvQN6Ix3gkdM/d3jSH+E0mcrvBqqVXTcGhEPcIwgErFYvJP9y1wYLOM7XamFNF3BqWu1Bcrly/+BnActe+3uIB/cMGX5oIbSvvnuMAhXBtnWYBRPIsGEuAYT+E0xtrApUmXxs3DjtmDC/73XMDfJxf8aN3csOL3w9NfkiuIV4v+l8Di81jA76b+iJszvBtEFB+16z1nbN7P5gN/vvdAcReufN+VP773QNzRacQDMh6Q8YCMrwYZxJ2eE/5o4dzwD9+SMrpBimwUbCVeE1vw/3tJCN5Bj/SQFwvgRGUv+Y7w8a5Nepc88jkobP+IFcsCVS6/BBIgoQu7LsORz3YFKPg/DdUmITcegNdrMbT2KHsQhifZFYWiaRnK4WQkc3CR+QVK0I2gd4KjDZEGDU03yYNi76uspCiS6CSDGEJnAy8UCH0k85CyYVvQ/JxCD2r7gYgI8pg+dB5dN3f+CibcYI3aIaXI7YNMXe0c1z5T7pu74Y/WO7V6Nsucft6N/RGkUMPA+kco9SHwc7gLxZ5TDHGkmvI1QoM7PIX4J8EZNP1hWyc3/N+3YBraZaCNJ1zMpWwWWhQaMo2F4xhOUAyLm5YA3fEfdRl/2ifiFg2AafMYTgMGowHkNZ4VkI9mWNcSaM7i/qxPfGDpFSz9vxDdzX7ccIkff1BxsyWfQPkfxV2fiCydFMQGMDNIDiSuQCduJWbm5O9ACxymON/lxpyLvC/gQs2rFtvpAKVTnS+zAV7ZxMpfBRdepjVN+/+By+tk/HMyGX+PZDjmcj9GeCYY4fWH+ymwYS4iSH4HbN7TxQfYPMDmPjr4AJsvCjY3Hedno837du4bo80IdgnsJqcw3+GZXzn/IW7CzB0HoF4NLHn6pfOjG+F6nxlLcgAPDH/GcZY930Y5wMZvHjFhBHNWLX9eQeK6sLkX+v2xh0t3FPn/cHZfJbL1cbj0OFy6tCD3RgL/ixCWm/PxFkAesbl7lTz15181huYnGejGtuMXjKfdz9wDRx448sCRB478Eo5QXxNH/kqQ2B3Hupztf9V27x/tegEfwykCeeXXk78fHSUFx9JYelbmCBHv2qz3j5QO5yVBtPtEDGn3WzyetODXjl1+8ejncAfFQYxHifskqaXIeMvBUOr0SrxR8RIR/rSNga8OPPiXipKSKIsT+FtemOMqQJJapV0bzHFRrCYRlIs+lD2REipRU2AVEtGCf7LwmqgsUNGxXd+wg36JCqtTXGvruijmE4aHpYhKH4nFcdvo4TUxy2mb1ZFgGhi4o4pdHqloqaC5bpLxfFcJXKpauIV/dNboJrkuhUbeqcBr0pKp91RtANpZMaKKXFMHk52KSyYg/OliKPdETen1gtJoQ6EstWWRryd1DYxgNlX3GNWY+/W+q8KO64S9SCRdTiu4qpbwd6Ir5IZpLypm0BAj1KRKT220pWgu+nrQ98TN1rOsueVS3RYBKEKIGIqjFtS4GxksYNoOZUHdk3iKirvkiJUswiKbvTlOhK6qJ32py7Lzio2PBjxn8A0czMMm2AyGulT2VEGO65wqVV2y4dTs7cDEk2nPLcO5BTsoGcgq4nG7hcxiHsq4aGkhawVV02C2fWrWahQOGPKhsx174zlOWzOuDPBJZgRRF5pSiQHo00okFhc7DD8UlXq7ACMejloALLE9WY2cfJVTRSVqClzbtX1WbLRU0K01lWamx9O2u6J7cnssZA0S3kvUrHnVN6OS0tTWmiH4mSyI0RLeoLsm+7xKzeEylQZjPACTrbidCuN2HNU5srDkrNLvyHmrO3OU2mpRU6amZrL1pcqtu7TTqfvAT9vJnCBncEUNoNuQpoDdrnkPdbw5tk1B7Kx8VWd9zuklOFgV7HBTh9l68L91M1s2p/NWY5sHy6UrdniaspF6tLduQ3bwnq3Jq5jwlorOVmB+seYsssRqDV1j1IHOW9vmfXYWNBtRoz+HN2HIqkMMqmTLYddBsSQTbZJsvSCZOSLfzfh+UwuqDaluMNkEuSuxlfFRuZlwW06XhVVebcpMzWyrVtPue7FLZbhRa4ZiLRDx1obU1wNVbjvhUBgQ/oRWAmlgpait/mjZWUSCVycb03pvLpUjZVhfOFy3KEYjvjopCwlhoc/W6usQVLmhsU0mpSupyF7qc51tqOUYrn5Jt+VGsk59rcfMminT39SJVmUjqvo8mQwcv6JalXG/38gnMzVVY0JsNlr6uNUXKo2pMlsRYFxXhAbXabD8Qhd1oTWRnHBeW9cmxso38y1faYR0Nc3NLGn1qelEXA91B94WFOuaHhDLidodsPik11HKmbFoym15qa79fLwRe0N61BjInlcfU100cRNPpPMu4cbyJOqjkUzNTZQTndGMd5nxkKddPTXYXk7wKZ3j1KhCoRUvkcgqsVV6m8U1tel3h9O+xsc8IcAFH42Bmof6yO9krYzuzleRPF5OV3LQTnxtLUWLJW2KfqcJq/B9KtVVq2ttdFv1+vZULAR6zavJ2OeWabPXtsab2nS5NKNE4qL+IBKTQqRkJWrOamWvUec7eLxaw2ZXZUiXOLKVvDPoRBsxyNctpcGmm1q6mAh0paUk8lqIPJpbONOIRRY63bqF2YAarI2qta67oDMgVaNkHQeBv1bb6dDo5L2s2tyWkUU63KI9abmBWFkNcjAMK6I8LxsrbxtkPhGYdUl1u1F3ELq1bdFoSVFbSchmzfLJSY9Z1VoDl16FOa+ZCgPbqGltqJXNuGTtBlUvx9OSdy3L1Yv+3MnG1UaV7HRF+HCSK6UkEzkqoTDwSUOb083Sm9rjSTFAphtvuk2PWoak1qFhum63RaXVnSyWlUxfLiYM9JU4MHhbKmOyTmijeS43mgNa1Ydr0pYDXOTpVujR1HA72AhotY76puRzUtdJ5gbL9LxhKXhSDVnzWlSph0mDZxsTW6/UdUOquQ1u7jJgCdh5ezsgjTDLjHjebwarqjARiihX3VEz6MZ2uRBbMejZZh0X9drERxZqWtv0ACFVRjgYw6QzD7cjuYWTVbtQ1htqPpTTAWM681XOJJ4yCHQuWe88oT+olxFYzzjHXCSyZ5dBZMRFXV60yaxDT7IONysDaWHolWW0VpxGpd7ppGpHVC0xs8t1Zx1IxgTU6qBm6oFAGUYnmc7QpPRb4mwctVqjDBqiLafmRr0TzUqHDVvLbMuPCEj22kDsKPgQcd1AZYOJsOmF45VHacKA06oLx5ddry26Fcub5QZiaCmiaU2zElig39WDCC2eZR7g2YAYSTRpbcTBWibIadP1pVEMwtk4FXUwmAX92owWe2ozUUmiNU3Z/iTizM4yEzdrCBbSWKdVL9ku5tuK2xB06E16m+qsl7Q8hU4sOSzw3ibUN0QHqYVVRRZmRIwDuT9tIp9Wq1vGKECGik3NydxNRB9PRpNlrnTiPkjwRqcggQWGqpso0sJbL4Wk4w6BXKMW2txVK66S8LUFnZZ9QVfTitWzFzpfGttwWGXEfNyesp1piq4JlorXV9yKEyxfnqzVtONHMdXgu2p96mhqoDfqihfW3SXZHhuQj0Z8xR4gB0FVDE8YWpuJJLVwI5p467Y2cCeKarQndS/XaGLNtZMRWfpmCLU3ZxazWXtSJaW6WMmQaVE2Oa1NhmpJMM0wHTEj4MgDdTvoCV5Hl1StP88B02fUFvLvgwgZo7oqWSbdZ3qiNE8H0yRhXHfTG5slOZA0SU7wfMnXlC4hgmnmV7MVWPRxLhas+tgB3TmswZ14GbUxaScJLX2jEGqvFxnJKIpYr63OemIzElbLxO9l/txgunVvzLjN7iZgek209SBum2SL8FxECyUf5dFEXW7RUkkc1t1WjAbytbSBazyC9TybOHSrzkq9UakDpdjk1jz1Cs1SWj7TdUS5xw0ZezFyx31yuBrAhy4JNlHbWHYk2/V61LR3cGgMhp1eg5EntdqVsKWTMKWnvxYeTT2TZ0dwu7Ciw2HfaSgR8XwQngcTPbPch+2J3dg8+RYbYo94okc80SOe6BPiiX68n/LJEUafsG38BSOMhl35nd2tVWr/1EHY7U+z24UC77wGI8F/cCJl/ImBV2SUeiaZC8FlmjsXEG9TqI5zwWWaOxcQl9UTF/cnLht4IniTOqsev7g/ftJA+O+d2PB3YsBPUKb0oU4bqbmzpyW0+T+Iyka12q9as6vjMih7nycMzTQPrJdSGdLyPFgBqGL7yvH3gre9LFmmu+bX7J0rf3v1H6REl1HrKPyd5nmoOOdx7odu345PD4FboBrhWKClvksp1JUIe1GUOIm/jI7/WwzIkBeB5RT9Bv+IY/D5Kf0Rx6CwP3ce+p0+ePCBfw/8e+Dfn8G/F5T4ZLC7bda+Mdud8bgBMmh1Q5DDNcSGyIdacGmxHvoLus0Qlt516CoLgtCELG8fXtWCtnwV2BcnnO/Guv3cS3NXOfEqK17jxavM+JYbz7LtSO7KHS6F12TcWyHxNtsR/t4Kr8muke5laeJKaeKi9Puc+d47iD/3siG84u5+7vGNw+sfi3BYSXs8hWibXwPV67r9k6h42/Z87C7iOUCyz/jpz1ucvPZiIncR/P8bMHkjtvVbkCQAvOlQFIdxOAAYDR9pMBNncYymKciSgIGe8919jgeG3HdQ2l+H6J/fZnt165dO/JMp7BNi3b8ggr3DU78ScM7chKlPCDh/+qUY8xt6cRZj/v77n18xxpz5Rp7wKz6Q3+WbO48Y80eM+aUFuTff+cMY8z//Hv+XijEfBqB8uvkC/wrmwJyX6z+gAPaHFNDc3/zUc7951L58Io8Cx9lDAjr1MV+Pg863ZZRbj8WHb3w8FH56/ebAHz8uMx//uIzhz6RAU2fPzNjxYOf3XqinmItaLzz5J7xRz34jvHgc2TyObB57JX/oyObSO93H0Q17n+zzidggh8nSGZnFbo+nufuGrKvwYKN85T7fT9ADd5MefuuDC+E1jaJpnjy5pgTZYUcKRXdkaGf68rBBkpEZ/zaHDSfz9feCUYiLWBRWeD4gzecdH9xS0AfYPMDmPjr4AJsvCja3zcu9gcXNrryDAl/6c3iMpbWvYjde+NuIFC0IkZ/+uNOTv/5dmk8/fXryhb6+SriPr84m7ug7Mn/FLN3pN/T9XzzF//584cE5v3DE9yW++enmErzwoqc+80vTwREH8tSMX0JUTylIXafQA++M8lt0aJqR5ZhoPJbxIRxmnwVp1UmN78BFeCh+yhTvqsqf20rheBWn/9tWioIzMiTc77KVcpiov7iNItzBPsoNK3YPNPTnwzBZkiAogsAckyLRHgOPmRZJYDjuECyLWyQLPvibCR9P4N9m/+jmRH0CP3z+/sKJZ73iR78UWaD7Jklx+sSJNKyF1i0U/gs=</diagram></mxfile>

typescript/opensearch/cwlogs_ingestion/lib/cwlogs_subscription_stack.ts

@@ -0,0 +1,99 @@
+import { PythonLayerVersion } from '@aws-cdk/aws-lambda-python-alpha';
+import { Duration, Stack, StackProps } from 'aws-cdk-lib';
+import { Rule, Schedule } from 'aws-cdk-lib/aws-events';
+import { LambdaFunction } from 'aws-cdk-lib/aws-events-targets';
+import { Effect, PolicyStatement } from 'aws-cdk-lib/aws-iam';
+import { Code, Runtime, Function, Alias } from 'aws-cdk-lib/aws-lambda';
+import { FilterPattern, LogGroup, RetentionDays, SubscriptionFilter } from 'aws-cdk-lib/aws-logs';
+import { LambdaDestination } from 'aws-cdk-lib/aws-logs-destinations';
+import { Construct } from 'constructs';
+import path = require('path');
+
+export interface CWLogsSubscriptionStackProps extends StackProps {
+  ingestionEndpointURL: string
+}
+
+export class CWLogsSubscriptionStack extends Stack {
+
+    private readonly STACK_NAMING_PREFIX: string = 'cwlogs-subscription';
+
+    constructor(scope: Construct, id: string, props: CWLogsSubscriptionStackProps) {
+      super(scope, id, props);
+
+        /////////////////////////////////////////////////////////////////////////////////
+        //
+        // Create the Log Emitter Lambda resources
+        // 
+        /////////////////////////////////////////////////////////////////////////////////
+        const logGroup = new LogGroup(this, `EventBridgeTriggeredLambdaLogGroup`, {
+          retention: RetentionDays.ONE_WEEK,
+        });
+
+        // Lambda Function to publish message
+        const lambdaFn = new Function(this, 'EventBridgeTriggeredLambdaFunction', {
+          code: Code.fromAsset(path.join(__dirname, '../resources/lambda/log_emitter')),
+          handler: 'handler.log_emitter',
+          timeout: Duration.seconds(300),
+          runtime: Runtime.PYTHON_3_12,
+          logGroup: logGroup
+        });
+
+        // Run the eventbridge every 5 minute interval to generate logs
+        const rule = new Rule(this, 'Rule', {
+          schedule: Schedule.rate(Duration.minutes(5))
+        });
+
+        // Add the lambda function as a target to the eventbridge
+        rule.addTarget(new LambdaFunction(lambdaFn));
+
+
+        /////////////////////////////////////////////////////////////////////////////////
+        //
+        // Create the CloudWatch Log group subscription filter resources
+        // 
+        /////////////////////////////////////////////////////////////////////////////////
+
+        const lambdaLayer = new PythonLayerVersion(this, `${this.STACK_NAMING_PREFIX}LambdaLayer`, {
+          entry: path.join(__dirname, "../resources/lambda/cw_subscription_filter/layers"),
+          compatibleRuntimes: [
+            Runtime.PYTHON_3_12,
+            Runtime.PYTHON_3_11,
+          ],
+          description: "A layer that contains the required modules",
+          license: "MIT License",
+        });
+
+        const lambdaFunction = new Function(this, `${this.STACK_NAMING_PREFIX}LambdaFunction`, {
+            runtime: Runtime.PYTHON_3_12,
+            code: Code.fromAsset(path.join(__dirname, '../resources/lambda/cw_subscription_filter')),
+            handler: 'handler.cw_subscription_handler',
+            layers: [lambdaLayer],
+            environment: {
+              OSI_INGESTION_ENDPOINT: props.ingestionEndpointURL,
+            },
+          }
+        );
+
+        new Alias(this, `${this.STACK_NAMING_PREFIX}LambdaFunctionAlias`, {
+          aliasName: 'live',
+          version: lambdaFunction.currentVersion,
+        });
+
+        lambdaFunction.addToRolePolicy(
+          new PolicyStatement({
+            effect: Effect.ALLOW,
+            resources: ['*'],
+            actions: ['osis:ingest'],
+          }),
+        );
+
+        // Create a Lambda Subscription Filter on the specific log group created above
+        const subscriptionFilter = new SubscriptionFilter(this, `${this.STACK_NAMING_PREFIX}LogSubscription`, {
+          logGroup: logGroup,
+          destination: new LambdaDestination(lambdaFunction),
+          filterPattern: FilterPattern.allEvents(),
+        });
+        subscriptionFilter.node.addDependency(lambdaFunction);
+      }
+
+}