Setup firehose stream to archive data

infracost-usage.yml

@@ -21,6 +21,25 @@ resource_usage:
       monthly_select_data_scanned_gb: 0 # Monthly data scanned by S3 Select in GB.
       monthly_select_data_returned_gb: 0 # Monthly data returned by S3 Select in GB.
       early_delete_gb: 0 # If an archive is deleted within 1 months of being uploaded, you will be charged an early deletion fee per GB.
+  module.ttl_archiver.aws_s3_bucket.this:
+    aws_s3_bucket:
+      object_tags: 50000 # Total object tags. Only for AWS provider V3.
+      standard: # Usages of S3 Standard:
+        storage_gb: 30 # Total storage in GB.
+        monthly_tier_1_requests: 10000 # Monthly PUT, COPY, POST, LIST requests (Tier 1).
+        monthly_tier_2_requests: 125000 # Monthly GET, SELECT, and all other requests (Tier 2).
+        monthly_select_data_scanned_gb: 20 # Monthly data scanned by S3 Select in GB.
+        monthly_select_data_returned_gb: 2 # Monthly data returned by S3 Select in GB.
+      intelligent_tiering: # Usages of S3 Intelligent - Tiering:
+        frequent_access_storage_gb: 30 # Total storage for Frequent Access Tier in GB.
+        infrequent_access_storage_gb: 0 # Total storage for Infrequent Access Tier in GB.
+        monitored_objects: 0 # Total objects monitored by the Intelligent Tiering.
+        monthly_tier_1_requests: 0 # Monthly PUT, COPY, POST, LIST requests (Tier 1).
+        monthly_tier_2_requests: 0 # Monthly GET, SELECT, and all other requests (Tier 2).
+        monthly_lifecycle_transition_requests: 0 # Monthly Lifecycle Transition requests.
+        monthly_select_data_scanned_gb: 0 # Monthly data scanned by S3 Select in GB.
+        monthly_select_data_returned_gb: 0 # Monthly data returned by S3 Select in GB.
+        early_delete_gb: 0 # If an archive is deleted within 1 months of being uploaded, you will be charged an early deletion fee per GB.
 
 resource_type_default_usage:
   aws_acmpca_certificate_authority:
@@ -204,7 +223,7 @@ resource_type_default_usage:
   aws_kinesisanalyticsv2_application_snapshot:
     durable_application_backup_gb: 208 # Total amount of durable application backups in GB.
   aws_kinesis_firehose_delivery_stream:
-    monthly_data_ingested_gb: 142 # Monthly data ingested by the Delivery Stream in GB.
+    monthly_data_ingested_gb: 20 # Monthly data ingested by the Delivery Stream in GB.
   aws_kinesis_stream:
     monthly_on_demand_data_in_gb: 62 # Monthly data ingested by the stream in GB.
     monthly_on_demand_data_out_gb: 125 # Monthly data egressed by the stream in GB total, (not per consumer application).
@@ -282,8 +301,8 @@ resource_type_default_usage:
       storage_gb: 10 # Total storage in GB.
       monthly_tier_1_requests: 10000 # Monthly PUT, COPY, POST, LIST requests (Tier 1).
       monthly_tier_2_requests: 125000 # Monthly GET, SELECT, and all other requests (Tier 2).
-      monthly_select_data_scanned_gb: 2500 # Monthly data scanned by S3 Select in GB.
-      monthly_select_data_returned_gb: 7250 # Monthly data returned by S3 Select in GB.
+      monthly_select_data_scanned_gb: 0 # Monthly data scanned by S3 Select in GB.
+      monthly_select_data_returned_gb: 0 # Monthly data returned by S3 Select in GB.
     intelligent_tiering: # Usages of S3 Intelligent - Tiering:
       frequent_access_storage_gb: 30 # Total storage for Frequent Access Tier in GB.
       infrequent_access_storage_gb: 0 # Total storage for Infrequent Access Tier in GB.

notebooks/.gitignore

@@ -0,0 +1,219 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+#poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+#pdm.lock
+#pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+#pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# Redis
+*.rdb
+*.aof
+*.pid
+
+# RabbitMQ
+mnesia/
+rabbitmq/
+rabbitmq-data/
+
+# ActiveMQ
+activemq-data/
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Abstra
+# Abstra is an AI-powered process automation framework.
+# Ignore directories containing user credentials, local state, and settings.
+# Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#  Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
+#  that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#  and can be added to the global gitignore or merged into this file. However, if you prefer,
+#  you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/
+
+# Streamlit
+.streamlit/secrets.toml
+
+# Config
+config.py

notebooks/README.md

@@ -0,0 +1,8 @@
+# Notebooks
+IPYNB notebooks for ACM tasks
+
+## Getting Started
+Create the file `config.py` with the following contents:
+```python
+AWS_PROFILE="<your authenticated AWS profile>"
+```

notebooks/read_archived_s3.ipynb

@@ -0,0 +1,84 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "ba97204d",
+   "metadata": {},
+   "source": [
+    "## Read Archived Logs from S3\n",
+    "Make sure that you have signed into AWS and setup a profile.\n",
+    "\n",
+    "Also, create `config.py` with contents:\n",
+    "```python\n",
+    "ARCHIVE_S3_BUCKET = \"<bucket name of archival data bucket>\"\n",
+    "AWS_PROFILE = \"<AWS_PROFILE HERE>\"\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "22f8f0a6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!pip install s3fs pandas"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "84a7370f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import config\n",
+    "\n",
+    "assert config.ARCHIVE_S3_BUCKET\n",
+    "assert config.AWS_PROFILE\n",
+    "module_to_read = \"ExampleTableWithStream\"\n",
+    "# configure this s3 path correctly based on the date you're looking for, etc.\n",
+    "s3_path = f\"s3://{config.ARCHIVE_S3_BUCKET}/resource={module_to_read}/**/*.gz\"\n",
+    "\n",
+    "df = pd.read_json(\n",
+    "    s3_path,\n",
+    "    lines=True,\n",
+    "    compression=\"gzip\",\n",
+    "    storage_options={\"profile\": config.AWS_PROFILE, \"expand\": True},\n",
+    ")\n",
+    "\n",
+    "print(df.head())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "328af1fa",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "base",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.13.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

package.json

@@ -5,13 +5,14 @@
   "type": "module",
   "workspaces": [
     "src/api",
-    "src/ui"
+    "src/ui",
+    "src/archival"
   ],
   "packageManager": "[email protected]",
   "scripts": {
     "postinstall": "npm run setup",
     "setup": "git config blame.ignoreRevsFile .git-blame-ignore-revs",
-    "build": "concurrently --names 'api,ui' 'yarn workspace infra-core-api run build' 'yarn workspace infra-core-ui run build'",
+    "build": "concurrently --names 'api,ui,archival' 'yarn workspace infra-core-api run build' 'yarn workspace infra-core-ui run build' 'yarn workspace infra-core-archival run build'",
     "postbuild": "node src/api/createLambdaPackage.js && yarn lockfile-manage",
     "dev": "cross-env DISABLE_AUDIT_LOG=true concurrently --names 'api,ui' 'yarn workspace infra-core-api run dev' 'yarn workspace infra-core-ui run dev'",
     "lockfile-manage": "synp --with-workspace --source-file yarn.lock",
@@ -38,6 +39,7 @@
     "@eslint/eslintrc": "^3.3.1",
     "@eslint/js": "^9.33.0",
     "@playwright/test": "^1.54.2",
+    "@smithy/types": "^4.3.2",
     "@tsconfig/node22": "^22.0.1",
     "@types/ioredis-mock": "^8.2.5",
     "@types/node": "^24.3.0",
@@ -92,4 +94,4 @@
     "pdfjs-dist": "^4.8.69",
     "form-data": "^4.0.4"
   }
-}
+}

src/api/build.js

@@ -1,3 +1,4 @@
+/* eslint-disable no-console */
 import esbuild from "esbuild";
 import { resolve } from "path";
 import { copy } from "esbuild-plugin-copy";