Create firehose stream and glue DB for audit logs

.github/workflows/deploy-prod.yml

@@ -103,11 +103,6 @@ jobs:
       - build
     environment: "AWS PROD"
     steps:
-      - name: Set up Node for testing
-        uses: actions/setup-node@v4
-        with:
-          node-version: 22.x
-
       - name: Setup Terraform
         uses: hashicorp/setup-terraform@v3
         with:
@@ -131,5 +126,6 @@ jobs:
         env:
           HUSKY: "0"
           VITE_RUN_ENVIRONMENT: prod
+
       - name: Call the health check script
         run: make prod_health_check

.github/workflows/deploy-qa.yml

@@ -106,25 +106,11 @@ jobs:
         env:
           HUSKY: "0"
 
-      - name: Set up Node for testing
-        uses: actions/setup-node@v4
-        with:
-          node-version: 22.x
-          cache: "yarn"
-
       - name: Setup Terraform
         uses: hashicorp/setup-terraform@v3
         with:
           terraform_version: 1.12.2
 
-      - name: Restore Yarn Cache
-        uses: actions/cache@v4
-        with:
-          path: node_modules
-          key: yarn-modules-${{ runner.arch }}-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}-dev
-          restore-keys: |
-            yarn-modules-${{ runner.arch }}-${{ runner.os }}-
-
       - name: Download Build files
         uses: actions/download-artifact@v4
         with:
@@ -143,6 +129,20 @@ jobs:
           HUSKY: "0"
           VITE_RUN_ENVIRONMENT: dev
 
+      - name: Set up Node for testing
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22.x
+          cache: "yarn"
+
+      - name: Restore Yarn Cache
+        uses: actions/cache@v4
+        with:
+          path: node_modules
+          key: yarn-modules-${{ runner.arch }}-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}-dev
+          restore-keys: |
+            yarn-modules-${{ runner.arch }}-${{ runner.os }}-
+
       - name: Run health check
         run: make dev_health_check
 

.github/workflows/manual-prod.yml

@@ -101,11 +101,6 @@ jobs:
       - build
     environment: "AWS PROD"
     steps:
-      - name: Setup Node
-        uses: actions/setup-node@v4
-        with:
-          node-version: 22.x
-
       - name: Setup Terraform
         uses: hashicorp/setup-terraform@v3
         with:
@@ -119,6 +114,7 @@ jobs:
         uses: actions/download-artifact@v4
         with:
           name: build-prod
+
       - uses: aws-actions/configure-aws-credentials@v4
         with:
           role-to-assume: arn:aws:iam::298118738376:role/GitHubActionsRole

notebooks/read_archived_s3.ipynb

@@ -22,7 +22,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!pip install s3fs pandas"
+    "%pip install s3fs pandas"
    ]
   },
   {

src/api/functions/auditLog.ts

@@ -6,6 +6,7 @@ import {
 import { marshall } from "@aws-sdk/util-dynamodb";
 import { genericConfig } from "common/config.js";
 import { AUDIT_LOG_RETENTION_DAYS } from "common/constants.js";
+import { ValidationError } from "common/errors/index.js";
 import { AuditLogEntry } from "common/types/logs.js";
 
 type AuditLogParams = {
@@ -69,3 +70,65 @@ export function buildAuditLogTransactPut({
     },
   };
 }
+
+/**
+ * Generates an efficient, partition-aware Athena SQL query for a given time range.
+ *
+ * @param startTs The start of the time range as a Unix timestamp in seconds.
+ * @param endTs The end of the time range as a Unix timestamp in seconds.
+ * @param moduleName The name of the module to query.
+ * @returns A SQL query string with WHERE clauses for partition pruning and predicate pushdown.
+ */
+export function buildAthenaQuery(
+  startTs: number,
+  endTs: number,
+  moduleName: string,
+): string {
+  if (startTs > endTs) {
+    throw new ValidationError({
+      message: "Start timestamp cannot be after end timestamp.",
+    });
+  }
+
+  const startDate = new Date(startTs * 1000);
+  const endDate = new Date(endTs * 1000);
+
+  const years = new Set<string>();
+  const months = new Set<string>();
+  const days = new Set<string>();
+  const hours = new Set<string>();
+
+  const currentDate = new Date(startDate);
+  while (currentDate <= endDate) {
+    // Extract UTC components to align with the Unix timestamp's nature
+    years.add(currentDate.getUTCFullYear().toString());
+
+    // Pad month, day, and hour with a leading zero if needed
+    months.add((currentDate.getUTCMonth() + 1).toString().padStart(2, "0"));
+    days.add(currentDate.getUTCDate().toString().padStart(2, "0"));
+    hours.add(currentDate.getUTCHours().toString().padStart(2, "0"));
+
+    // Move to the next hour
+    currentDate.setUTCHours(currentDate.getUTCHours() + 1);
+  }
+
+  const createInClause = (valueSet: Set<string>): string => {
+    return Array.from(valueSet)
+      .sort()
+      .map((value) => `'${value}'`)
+      .join(", ");
+  };
+
+  const query = `
+SELECT *
+FROM "logs"
+WHERE
+  module = "${moduleName}"
+  AND year IN (${createInClause(years)})
+  AND month IN (${createInClause(months)})
+  AND day IN (${createInClause(days)})
+  AND hour IN (${createInClause(hours)})
+  AND createdAt BETWEEN ${startTs} AND ${endTs};
+`;
+  return query.trim();
+}

terraform/envs/prod/variables.tf

@@ -42,3 +42,8 @@ variable "IcalPublicDomain" {
   type    = string
   default = "ical.acm.illinois.edu"
 }
+
+variable "AuditLogRetentionDays" {
+  type    = number
+  default = 1430
+}