Update to version v2.3.1

Author: jangidms

.github/workflows/gh-page.yml

@@ -0,0 +1,40 @@
+name: GitHub Pages
+
+on:
+  push:
+    branches:
+      - main # Set a branch name to trigger deployment
+  pull_request:
+
+jobs:
+  deploy:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # Fetch all history for .GitInfo and .Lastmod
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.9"
+
+      - name: Install dependencies
+        run: |
+          python3 -m pip install mkdocs==1.3.1            # install mkdocs
+          python3 -m pip install mkdocs-material==8.5.3   # install material theme
+          python3 -m pip install mkdocs-macros-plugin==0.7.0  # install macros plugin
+          python3 -m pip install mkdocs-include-markdown-plugin==3.8.1 # install include-markdown
+
+      - name: Build mkdocs
+        run: |
+          mkdocs build -f ./docs/mkdocs.en.yml
+          mkdocs build -f ./docs/mkdocs.zh.yml
+          cp -av ./docs/index.html ./docs/site
+
+      - name: Deploy
+        uses: peaceiris/actions-gh-pages@v3
+        if: ${{ github.ref == 'refs/heads/main' }}
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./docs/site

.gitignore

@@ -1,6 +1,8 @@
+# Typescript
 source/*/*.js
 *.d.ts
 *.js
+!.nightswatch/**/*.js
 node_modules/
 
 # CDK asset staging directory
@@ -13,21 +15,19 @@ cdk.out
 # Jetbrains project
 .idea/
 
-# mkdocs documentation
-docs/site/
-
 # MaxMind GeoIP database
 source/constructs/lambda/plugin/standard/assets/GeoLite2-City.mmdb
 source/constructs/lambda/microbatch/utils/enrichment/maxminddb/GeoLite2-City.mmdb
 
 # Test folder including coverage report
 source/tests/
+source/test/
 
 # Deployment
-staging
-global-s3-assets
-regional-s3-assets
-viperlight
+deployment/global-s3-assets/
+deployment/regional-s3-assets/
+deployment/viperlight
+deployment/staging/
 
 ### macOS ###
 # General
@@ -49,7 +49,7 @@ __pycache__/
 pip-log.txt
 pip-delete-this-directory.txt
 
-## Distribution / packaging
+# Distribution / packaging
 *.egg-info/
 
 # Unit test / coverage reports
@@ -78,3 +78,7 @@ ENV/
 env.bak/
 venv.bak/
 
+*.mmdb
+
+#requirement files
+source/constructs/**/*requirement*.txt

CHANGELOG.md

@@ -5,7 +5,24 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [2.3.0] - 2024-12-17
+## [2.3.1] - 2025-02-24
+
+### Changed
+
+- Migrated to Poetry for Python dependency management
+
+### Fixed
+
+- Fixed S3 permission issue when creating cross-account Lambda log ingestion pipeline [Issue #312](https://github.com/aws-solutions/centralized-logging-with-opensearch/issues/312)
+- Fixed STS credential expiration handling when ingesting logs from cross-account sources
+- Fixed Opensearch index rollover timeout issue
+
+### Security
+
+- Updated serialize-javascript package to address [CVE-2024-11831](https://avd.aquasec.com/nvd/2024/cve-2024-11831/)
+- Updated cryptography package to address [CVE-2024-12797](https://avd.aquasec.com/nvd/2024/cve-2024-12797/)
+
+## [2.3.0] - 2024-12-11
 
 ### Added
 
@@ -26,7 +43,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fix the issue that the EKS DaemonSet Guide is not refreshed after editing the log config. #241
 - Fix the user interface issue that a wrong S3 bucket prefix is using when creating WAF log pipeline if the logging has already been enabled. #67
 
-
 ## [2.2.2] - 2024-08-23
 
 ### Changed

CUSTOM_BUILD.md

@@ -10,8 +10,22 @@ Before you start customizing the solution, make sure you have the following prer
 - Python (>=3.9)
 - NodeJS (v18 or later)
 - Docker
-
 > if you are using ARM CPU like Apple M1 chip, please run `export DOCKER_DEFAULT_PLATFORM=linux/amd64` for building amd64 container image.
+-   Install Poetry. Below is one of the ways to install poetry. For other ways to install poetry, refer [Poetry installation instructions](https://python-poetry.org/docs/#installation)
+
+```shell
+## Install pipx via pip
+python3 -m pip install --user pipx
+python3 -m pipx ensurepath
+
+## OR Install pipx via brew
+brew install pipx
+pipx ensurepath
+
+## Install poetry
+pipx install poetry
+pipx inject poetry poetry-plugin-export
+```
 
 Clone the repository and make desired code changes.
 
@@ -29,6 +43,7 @@ The following is the file structure of the solution. You can customize the solut
 │   ├── cdk-solution-helper/                - helper function for converting CDK output to a format compatible with the AWS Solutions pipelines.
 │   ├── build-open-source-dist.sh           - builds the open source package with cleaned assets and builds a .zip file in the /open-source folder for distribution to GitHub
 │   ├── build-s3-dist.sh                    - builds the solution and copies artifacts to the appropriate /global-s3-assets or /regional-s3-assets folders.
+│   ├── run-unit-tests.sh                    - runs all the unit tests within the /source folder
 ├── source/
 │   ├── constructs
 │   │   ├── bin
@@ -58,9 +73,9 @@ The following is the file structure of the solution. You can customize the solut
 After you have customized the solution. Run the unit tests to ensure the solution is working as expected. Review the generated coverage report.
 
 ```
-cd ./source
-chmod +x ./run-all-tests.sh
-./run-all-tests.sh
+cd ./deployment
+chmod +x ./run-unit-tests.sh
+./run-unit-tests.sh
 cd ..
 ```
 

NOTICE.txt

@@ -1717,6 +1717,28 @@ yocto-queue under the MIT license
 zen-observable under the MIT license
 zen-observable-ts under the MIT license
 zen-push under the MIT license
+urlgrabber under the LGPL-2.0-or-later license.
+pygpgme under the LGPL-2.1-only license.
+pycurl under the MIT license.
+call-bind-apply-helpers under the MIT license.
+dunder-proto under the MIT license.
+highlightjs-vue under the CC0-1.0 license.
+@rtsao/scc under the MIT license.
+regjsgen under the MIT license.
+@babel/plugin-transform-regexp-modifiers under the MIT license.
+color-support under the ISC license.
+@aws-solutions-constructs/resources under the Apache-2.0 license.
+charset-normalizer under the MIT license.
+python-dateutil under the Apache-2.0 license.
+six under the MIT license.
+typing-extensions under the PSF-2.0 license.
+jinja2 under the 0BSD license.
+markupsafe under the 0BSD license.
+packaging under the Apache-2.0 license.
+pywin32 under the PSF-2.0 license.
+werkzeug under the 0BSD license.
+commonlib under the 0BSD license.
+pyopenssl under the Apache-2.0 license.
 
 
 ********************
@@ -1738,4 +1760,5 @@ Python-2.0 - https://opensource.org/licenses/Python-2.0
 Unlicense - https://opensource.org/licenses/Unlicense
 LGPL-2.0-or-later - https://spdx.org/licenses/GPL-2.0-or-later.html
 LGPL-2.1-only - https://spdx.org/licenses/LGPL-2.1-only.html
-Zlib - https://spdx.org/licenses/Zlib.html
+Zlib - https://spdx.org/licenses/Zlib.html
+PSF-2.0 - https://opensource.org/licenses/Python-2.0

arch.png

@@ -73,6 +73,15 @@ do_cmd()
     fi
 }
 
+if command -v poetry >/dev/null 2>&1; then
+    export POETRY_COMMAND="poetry"
+elif [ -n "$POETRY_HOME" ] && [ -x "$POETRY_HOME/bin/poetry" ]; then
+    export POETRY_COMMAND="$POETRY_HOME/bin/poetry"
+else
+    echo "Poetry is not available. Aborting script." >&2
+    exit 1
+fi
+
 sedi()
 {
     # cross-platform for sed -i
@@ -284,9 +293,37 @@ t do_cmd npm run build       # build javascript from typescript to validate the
 
 
 echo "------------------------------------------------------------------------------"
-echo "${bold}[Create] Templates${normal}"
+echo "[Install] Install dependencies for Lambda functions & layers"
 echo "------------------------------------------------------------------------------"
 
+do_cmd cd $source_dir/constructs/lib/microbatch/main/services/lambda/layer
+"$POETRY_COMMAND" export --format requirements.txt --output requirements-boto3.txt --without-hashes --only boto3
+"$POETRY_COMMAND" export --format requirements.txt --output requirements-pyarrow.txt --without-hashes --only pyarrow
+"$POETRY_COMMAND" export --format requirements.txt --output requirements-utils.txt --without-hashes --only utils
+"$POETRY_COMMAND" export --format requirements.txt --output requirements-enrichment.txt --without-hashes --only enrichment
+
+lambda_paths=(
+    "common-lib"
+    "api/app_log_ingestion"
+    "api/app_pipeline"
+    "api/cluster"
+    "api/log_source"
+    "plugin/standard"
+    "api/pipeline_ingestion_flow"
+)
+
+base_lambda_dir="$source_dir/constructs/lambda"
+for path in "${lambda_paths[@]}"; do
+    full_path="$base_lambda_dir/$path"
+    do_cmd cd "$full_path"
+    "$POETRY_COMMAND" export --format requirements.txt --output requirements.txt --without-hashes --without dev
+done
+
+
+echo "------------------------------------------------------------------------------"
+echo "${bold}[Create] Templates${normal}"
+echo "------------------------------------------------------------------------------"
+do_cmd cd $source_dir/constructs
 if fn_exists create_template_${template_format}; then
     t create_template_${template_format}
 else
@@ -375,3 +412,18 @@ cd $template_dir
 # build ecr
 echo "Run s3_list_objects_dir/build.sh"
 t do_cmd $s3_list_objects_dir/build.sh
+
+# cleanup requirement.txt files 
+paths=(
+    "$source_dir/constructs/lib/microbatch/main/services/lambda/layer"
+    "$source_dir/constructs/lambda/common-lib"
+    "$source_dir/constructs/lambda/api/app_log_ingestion"
+    "$source_dir/constructs/lambda/api/app_pipeline"
+    "$source_dir/constructs/lambda/api/cluster"
+    "$source_dir/constructs/lambda/api/log_source"
+    "$source_dir/constructs/lambda/plugin/standard"
+    "$source_dir/constructs/lambda/api/pipeline_ingestion_flow"
+)
+for path in "${paths[@]}"; do
+    rm $path/requirements*.txt
+done

deployment/build-s3-dist.sh

@@ -1,27 +1,27 @@
-FROM public.ecr.aws/lambda/python:3.11.2024.11.22.15 AS builder
+FROM public.ecr.aws/lambda/python:3.12.2025.01.24.11 AS builder
 
 WORKDIR /build
 
-COPY common-lib .
+COPY common-lib ./common-lib
+COPY pyproject.toml poetry.lock ./
 
 RUN python -m venv .venv && \
     source .venv/bin/activate && \
     pip install --upgrade build && \
-    pip install --upgrade pip && \
-    pip install setuptools==70.0.0 && \
-    python3 -m build -s
+    pip install poetry==2.0.1 && \
+    poetry self add poetry-plugin-export && \
+    poetry export --format requirements.txt --output requirements.txt --without-hashes --without dev && \
+    cd common-lib && \
+    poetry build
 
-
-FROM public.ecr.aws/lambda/python:3.11.2024.11.22.15
+FROM public.ecr.aws/lambda/python:3.12.2025.01.24.11
 
 WORKDIR /ws
 
-COPY requirements.txt .
-COPY --from=builder /build/dist/commonlib-0.1.0.tar.gz .
+COPY --from=builder /build/common-lib/dist/commonlib-0.1.0.tar.gz .
+COPY --from=builder /build/requirements.txt .
 
 RUN pip install --no-cache-dir -r requirements.txt && \
-    pip install --upgrade pip && \
-    pip install setuptools==70.0.0 && \
     pip install --no-cache-dir commonlib-0.1.0.tar.gz
 
 COPY . .