diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml index f3391372..f36f46ff 100644 --- a/.github/workflows/deploy-prod.yml +++ b/.github/workflows/deploy-prod.yml @@ -103,11 +103,6 @@ jobs: - build environment: "AWS PROD" steps: - - name: Set up Node for testing - uses: actions/setup-node@v4 - with: - node-version: 22.x - - name: Setup Terraform uses: hashicorp/setup-terraform@v3 with: @@ -131,5 +126,6 @@ jobs: env: HUSKY: "0" VITE_RUN_ENVIRONMENT: prod + - name: Call the health check script run: make prod_health_check diff --git a/.github/workflows/deploy-qa.yml b/.github/workflows/deploy-qa.yml index 1ecd2ff0..b5b4a12f 100644 --- a/.github/workflows/deploy-qa.yml +++ b/.github/workflows/deploy-qa.yml @@ -106,25 +106,11 @@ jobs: env: HUSKY: "0" - - name: Set up Node for testing - uses: actions/setup-node@v4 - with: - node-version: 22.x - cache: "yarn" - - name: Setup Terraform uses: hashicorp/setup-terraform@v3 with: terraform_version: 1.12.2 - - name: Restore Yarn Cache - uses: actions/cache@v4 - with: - path: node_modules - key: yarn-modules-${{ runner.arch }}-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}-dev - restore-keys: | - yarn-modules-${{ runner.arch }}-${{ runner.os }}- - - name: Download Build files uses: actions/download-artifact@v4 with: @@ -143,6 +129,20 @@ jobs: HUSKY: "0" VITE_RUN_ENVIRONMENT: dev + - name: Set up Node for testing + uses: actions/setup-node@v4 + with: + node-version: 22.x + cache: "yarn" + + - name: Restore Yarn Cache + uses: actions/cache@v4 + with: + path: node_modules + key: yarn-modules-${{ runner.arch }}-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}-dev + restore-keys: | + yarn-modules-${{ runner.arch }}-${{ runner.os }}- + - name: Run health check run: make dev_health_check diff --git a/.github/workflows/manual-prod.yml b/.github/workflows/manual-prod.yml index c99f3626..4caaab33 100644 --- a/.github/workflows/manual-prod.yml +++ b/.github/workflows/manual-prod.yml @@ -101,11 +101,6 @@ jobs: - build environment: "AWS PROD" steps: - - name: Setup Node - uses: actions/setup-node@v4 - with: - node-version: 22.x - - name: Setup Terraform uses: hashicorp/setup-terraform@v3 with: @@ -119,6 +114,7 @@ jobs: uses: actions/download-artifact@v4 with: name: build-prod + - uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: arn:aws:iam::298118738376:role/GitHubActionsRole diff --git a/notebooks/read_archived_s3.ipynb b/notebooks/read_archived_s3.ipynb index be83569d..be1e6a61 100644 --- a/notebooks/read_archived_s3.ipynb +++ b/notebooks/read_archived_s3.ipynb @@ -22,7 +22,7 @@ "metadata": {}, "outputs": [], "source": [ - "!pip install s3fs pandas" + "%pip install s3fs pandas" ] }, { diff --git a/src/api/functions/auditLog.ts b/src/api/functions/auditLog.ts index 7322a8bf..cdfd8a11 100644 --- a/src/api/functions/auditLog.ts +++ b/src/api/functions/auditLog.ts @@ -6,6 +6,7 @@ import { import { marshall } from "@aws-sdk/util-dynamodb"; import { genericConfig } from "common/config.js"; import { AUDIT_LOG_RETENTION_DAYS } from "common/constants.js"; +import { ValidationError } from "common/errors/index.js"; import { AuditLogEntry } from "common/types/logs.js"; type AuditLogParams = { @@ -69,3 +70,65 @@ export function buildAuditLogTransactPut({ }, }; } + +/** + * Generates an efficient, partition-aware Athena SQL query for a given time range. + * + * @param startTs The start of the time range as a Unix timestamp in seconds. + * @param endTs The end of the time range as a Unix timestamp in seconds. + * @param moduleName The name of the module to query. + * @returns A SQL query string with WHERE clauses for partition pruning and predicate pushdown. + */ +export function buildAthenaQuery( + startTs: number, + endTs: number, + moduleName: string, +): string { + if (startTs > endTs) { + throw new ValidationError({ + message: "Start timestamp cannot be after end timestamp.", + }); + } + + const startDate = new Date(startTs * 1000); + const endDate = new Date(endTs * 1000); + + const years = new Set(); + const months = new Set(); + const days = new Set(); + const hours = new Set(); + + const currentDate = new Date(startDate); + while (currentDate <= endDate) { + // Extract UTC components to align with the Unix timestamp's nature + years.add(currentDate.getUTCFullYear().toString()); + + // Pad month, day, and hour with a leading zero if needed + months.add((currentDate.getUTCMonth() + 1).toString().padStart(2, "0")); + days.add(currentDate.getUTCDate().toString().padStart(2, "0")); + hours.add(currentDate.getUTCHours().toString().padStart(2, "0")); + + // Move to the next hour + currentDate.setUTCHours(currentDate.getUTCHours() + 1); + } + + const createInClause = (valueSet: Set): string => { + return Array.from(valueSet) + .sort() + .map((value) => `'${value}'`) + .join(", "); + }; + + const query = ` +SELECT * +FROM "logs" +WHERE + module = "${moduleName}" + AND year IN (${createInClause(years)}) + AND month IN (${createInClause(months)}) + AND day IN (${createInClause(days)}) + AND hour IN (${createInClause(hours)}) + AND createdAt BETWEEN ${startTs} AND ${endTs}; +`; + return query.trim(); +} diff --git a/terraform/envs/prod/variables.tf b/terraform/envs/prod/variables.tf index 7edf08cd..194b65a5 100644 --- a/terraform/envs/prod/variables.tf +++ b/terraform/envs/prod/variables.tf @@ -42,3 +42,8 @@ variable "IcalPublicDomain" { type = string default = "ical.acm.illinois.edu" } + +variable "AuditLogRetentionDays" { + type = number + default = 1430 +} diff --git a/terraform/modules/auditlog/main.tf b/terraform/modules/auditlog/main.tf new file mode 100644 index 00000000..4d899553 --- /dev/null +++ b/terraform/modules/auditlog/main.tf @@ -0,0 +1,266 @@ + +resource "null_resource" "onetime_auditlog" { + provisioner "local-exec" { + command = <<-EOT + set -e + python auditlog-migration.py + EOT + interpreter = ["bash", "-c"] + working_dir = "${path.module}/../../../onetime/" + } +} + +data "aws_caller_identity" "current" {} +data "aws_region" "current" {} + +locals { + firehose_stream_name = "${var.ProjectId}-audit-log-stream" + glue_db_name = "${replace(var.ProjectId, "-", "_")}_audit_logs" + glue_table_name = "logs" + s3_bucket_name = "${var.BucketPrefix}-audit-logs" +} + +# 1. S3 Bucket and Configuration (No changes) +resource "aws_s3_bucket" "this" { + bucket = local.s3_bucket_name +} + +resource "aws_s3_bucket_versioning" "this" { + bucket = aws_s3_bucket.this.id + versioning_configuration { + status = "Enabled" + } +} + +resource "aws_s3_bucket_lifecycle_configuration" "this" { + bucket = aws_s3_bucket.this.id + + rule { + id = "AbortIncompleteMultipartUploads" + status = "Enabled" + abort_incomplete_multipart_upload { + days_after_initiation = 1 + } + } + rule { + id = "intelligent-tiering-transition" + status = "Enabled" + filter {} + transition { + days = 1 + storage_class = "INTELLIGENT_TIERING" + } + } + rule { + id = "ExpireNoncurrentVersions" + status = "Enabled" + filter {} + noncurrent_version_expiration { + noncurrent_days = 5 + } + } + rule { + id = "DeleteAuditLogsAfterDays" + status = "Enabled" + filter {} + expiration { + days = var.DataExpirationDays + } + } +} + +resource "aws_s3_bucket_intelligent_tiering_configuration" "this" { + bucket = aws_s3_bucket.this.id + name = "ArchiveAfterSixMonths" + status = "Enabled" + tiering { + access_tier = "ARCHIVE_ACCESS" + days = 180 + } +} + +resource "aws_cloudwatch_log_group" "firehose_logs" { + name = "/aws/kinesisfirehose/${local.firehose_stream_name}" + retention_in_days = var.LogRetentionDays +} + +resource "aws_cloudwatch_log_stream" "firehose_logs_stream" { + log_group_name = aws_cloudwatch_log_group.firehose_logs.name + name = "DataArchivalS3Delivery" +} + +# 3. AWS Glue Catalog for Parquet Conversion (No changes) +resource "aws_glue_catalog_table" "this" { + name = local.glue_table_name + database_name = aws_glue_catalog_database.this.name + table_type = "EXTERNAL_TABLE" + parameters = { + "EXTERNAL" = "TRUE" + "parquet.compression" = "SNAPPY" + } + + storage_descriptor { + location = "s3://${aws_s3_bucket.this.id}/" + input_format = "org.apache.hadoop.mapred.TextInputFormat" + output_format = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat" + ser_de_info { + name = "parquet-serde" + serialization_library = "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe" + parameters = { "serialization.format" = "1" } + } + + columns { + name = "createdAt" + type = "bigint" + } + columns { + name = "actor" + type = "string" + } + columns { + name = "message" + type = "string" + } + columns { + name = "requestId" + type = "string" + } + columns { + name = "target" + type = "string" + } + } + partition_keys { + name = "module" + type = "string" + } + partition_keys { + name = "year" + type = "string" + } + partition_keys { + name = "month" + type = "string" + } + partition_keys { + name = "day" + type = "string" + } + partition_keys { + name = "hour" + type = "string" + } +} + +resource "aws_glue_catalog_database" "this" { + name = local.glue_db_name +} + + +resource "aws_iam_role" "firehose_role" { + name = "${local.firehose_stream_name}-exec-role" + assume_role_policy = jsonencode({ + Version = "2012-10-17", + Statement = [{ + Action = "sts:AssumeRole", + Effect = "Allow", + Principal = { Service = "firehose.amazonaws.com" } + }] + }) +} + +resource "aws_iam_policy" "firehose_policy" { + name = "${local.firehose_stream_name}-s3-policy" + policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Effect = "Allow", + Action = [ + "s3:AbortMultipartUpload", "s3:GetBucketLocation", "s3:GetObject", + "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:PutObject" + ], + Resource = [aws_s3_bucket.this.arn, "${aws_s3_bucket.this.arn}/*"] + }, + { + Effect = "Allow", + Action = ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"], + Resource = [aws_cloudwatch_log_group.firehose_logs.arn] + }, + { + Effect = "Allow", + Action = ["glue:GetTable", "glue:GetTableVersion", "glue:GetTableVersions"], + Resource = [ + "arn:aws:glue:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:catalog", + aws_glue_catalog_database.this.arn, + aws_glue_catalog_table.this.arn + ] + } + ] + }) +} + +resource "aws_iam_role_policy_attachment" "firehose_attach" { + role = aws_iam_role.firehose_role.name + policy_arn = aws_iam_policy.firehose_policy.arn +} + +resource "aws_kinesis_firehose_delivery_stream" "dynamic_stream" { + name = local.firehose_stream_name + destination = "extended_s3" + + extended_s3_configuration { + bucket_arn = aws_s3_bucket.this.arn + role_arn = aws_iam_role.firehose_role.arn + compression_format = "UNCOMPRESSED" + buffering_interval = 60 + buffering_size = 128 + + data_format_conversion_configuration { + enabled = true + input_format_configuration { + deserializer { + open_x_json_ser_de {} + } + } + output_format_configuration { + serializer { + parquet_ser_de {} + } + } + schema_configuration { + database_name = aws_glue_catalog_database.this.name + table_name = aws_glue_catalog_table.this.name + role_arn = aws_iam_role.firehose_role.arn + } + } + + processing_configuration { + enabled = true + processors { + type = "MetadataExtraction" + parameters { + parameter_name = "MetadataExtractionQuery" + parameter_value = "{module: .module, year: (.createdAt | strftime(\"%Y\")), month: (.createdAt | strftime(\"%m\")), day: (.createdAt | strftime(\"%d\")), hour: (.createdAt | strftime(\"%H\"))}" + } + parameters { + parameter_name = "JsonParsingEngine" + parameter_value = "JQ-1.6" + } + } + } + + dynamic_partitioning_configuration { + enabled = true + } + + cloudwatch_logging_options { + enabled = true + log_group_name = aws_cloudwatch_log_group.firehose_logs.name + log_stream_name = aws_cloudwatch_log_stream.firehose_logs_stream.name + } + + prefix = "module=!{partitionKeyFromQuery:module}/year=!{partitionKeyFromQuery:year}/month=!{partitionKeyFromQuery:month}/day=!{partitionKeyFromQuery:day}/hour=!{partitionKeyFromQuery:hour}/" + error_output_prefix = "firehose-errors/!{firehose:error-output-type}/!{timestamp:yyyy/MM/dd}/" + } +} diff --git a/terraform/modules/auditlog/outputs.tf b/terraform/modules/auditlog/outputs.tf new file mode 100644 index 00000000..71c810f1 --- /dev/null +++ b/terraform/modules/auditlog/outputs.tf @@ -0,0 +1,24 @@ +output "firehose_delivery_stream_name" { + description = "The name of the Kinesis Firehose delivery stream." + value = aws_kinesis_firehose_delivery_stream.dynamic_stream.name +} + +output "firehose_delivery_stream_arn" { + description = "The ARN of the Kinesis Firehose delivery stream." + value = aws_kinesis_firehose_delivery_stream.dynamic_stream.arn +} + +output "s3_bucket_name" { + description = "The name of the S3 bucket where data is stored." + value = aws_s3_bucket.this.bucket +} + +output "glue_database_name" { + description = "The name of the AWS Glue database." + value = aws_glue_catalog_database.this.name +} + +output "glue_table_name" { + description = "The name of the AWS Glue table." + value = aws_glue_catalog_table.this.name +} diff --git a/terraform/modules/auditlog/variables.tf b/terraform/modules/auditlog/variables.tf new file mode 100644 index 00000000..b53c5de1 --- /dev/null +++ b/terraform/modules/auditlog/variables.tf @@ -0,0 +1,27 @@ +variable "ProjectId" { + type = string + description = "Prefix before each resource" +} + +variable "BucketPrefix" { + type = string +} + + +variable "LogRetentionDays" { + type = number +} + +variable "DataExpirationDays" { + type = number +} + + +variable "RunEnvironment" { + type = string + validation { + condition = var.RunEnvironment == "dev" || var.RunEnvironment == "prod" + error_message = "The lambda run environment must be dev or prod." + } +} +