Create a seperate slow lambda for slow API requests (#248)

This will prevent clogging up our alarms

Author: devksingh4

src/api/lambda.ts

@@ -10,7 +10,41 @@ const realHandler = awsLambdaFastify(app, {
   callbackWaitsForEmptyEventLoop: false,
   binaryMimeTypes: ["application/octet-stream", "application/vnd.apple.pkpass"],
 });
+
 type WarmerEvent = { action: "warmer" };
+
+/**
+ * Validates the origin verification header against the current and previous keys.
+ * @returns {boolean} `true` if the request is valid, otherwise `false`.
+ */
+const validateOriginHeader = (
+  originHeader: string | undefined,
+  currentKey: string,
+  previousKey: string | undefined,
+  previousKeyExpiresAt: string | undefined,
+): boolean => {
+  // 1. A header must exist to be valid.
+  if (!originHeader) {
+    return false;
+  }
+
+  // 2. Check against the current key first for an early return on the happy path.
+  if (originHeader === currentKey) {
+    return true;
+  }
+
+  // 3. If it's not the current key, check the previous key during the rotation window.
+  if (previousKey && previousKeyExpiresAt) {
+    const isExpired = new Date() >= new Date(previousKeyExpiresAt);
+    if (originHeader === previousKey && !isExpired) {
+      return true;
+    }
+  }
+
+  // 4. If all checks fail, the header is invalid.
+  return false;
+};
+
 const handler = async (
   event: APIGatewayEvent | WarmerEvent,
   context: Context,
@@ -19,12 +53,32 @@ const handler = async (
     return { instanceId };
   }
   event = event as APIGatewayEvent;
-  if (process.env.ORIGIN_VERIFY_KEY) {
-    // check that the request has the right header (coming from cloudfront)
-    if (
-      !event.headers ||
-      !(event.headers["x-origin-verify"] === process.env.ORIGIN_VERIFY_KEY)
-    ) {
+
+  const currentKey = process.env.ORIGIN_VERIFY_KEY;
+  const previousKey = process.env.PREVIOUS_ORIGIN_VERIFY_KEY;
+  const previousKeyExpiresAt =
+    process.env.PREVIOUS_ORIGIN_VERIFY_KEY_EXPIRES_AT;
+
+  // Log an error if the previous key has expired but is still configured.
+  if (previousKey && previousKeyExpiresAt) {
+    if (new Date() >= new Date(previousKeyExpiresAt)) {
+      console.error(
+        "Expired previous origin verify key is still present in the environment. Expired at:",
+        previousKeyExpiresAt,
+      );
+    }
+  }
+
+  // Proceed with verification only if a current key is set.
+  if (currentKey) {
+    const isValid = validateOriginHeader(
+      event.headers?.["x-origin-verify"],
+      currentKey,
+      previousKey,
+      previousKeyExpiresAt,
+    );
+
+    if (!isValid) {
       const newError = new ValidationError({
         message: "Request is not valid.",
       });
@@ -38,9 +92,11 @@ const handler = async (
         isBase64Encoded: false,
       };
     }
+
     delete event.headers["x-origin-verify"];
   }
-  // else proceed with handler logic
+
+  // If verification is disabled or passed, proceed with the real handler logic.
   return await realHandler(event, context).catch((e) => {
     console.error(e);
     const newError = new InternalServerError({
@@ -58,5 +114,5 @@ const handler = async (
   });
 };
 
-await app.ready(); // needs to be placed after awsLambdaFastify call because of the decoration: https://github.com/fastify/aws-lambda-fastify/blob/master/index.js#L9
+await app.ready();
 export { handler };

terraform/envs/prod/.terraform.lock.hcl

@@ -45,21 +45,14 @@ module "sqs_queues" {
   core_sqs_consumer_lambda_name = module.lambdas.core_sqs_consumer_lambda_name
 }
 
-module "lambda_warmer" {
-  source           = "github.com/acm-uiuc/terraform-modules/lambda-warmer?ref=v1.0.1"
-  function_to_warm = module.lambdas.core_api_lambda_name
-}
 module "dynamo" {
   source    = "../../modules/dynamo"
   ProjectId = var.ProjectId
 }
 
-resource "random_password" "origin_verify_key" {
-  length  = 16
-  special = false
-  keepers = {
-    force_recreation = formatdate("DD-MMM-YYYY", plantimestamp())
-  }
+module "origin_verify" {
+  source    = "../../modules/origin_verify"
+  ProjectId = var.ProjectId
 }
 
 resource "aws_cloudfront_key_value_store" "linkry_kv" {
@@ -72,27 +65,31 @@ module "alarms" {
   resource_prefix                 = var.ProjectId
   main_cloudfront_distribution_id = module.frontend.main_cloudfront_distribution_id
   standard_sns_arn                = var.GeneralSNSAlertArn
-  main_lambda_function_name       = module.lambdas.core_api_lambda_name
+  all_lambdas                     = toset([module.lambdas.core_api_lambda_name, module.lambdas.core_api_slow_lambda_name, module.lambdas.core_sqs_consumer_lambda_name])
+  performance_noreq_lambdas       = toset([module.lambdas.core_api_lambda_name])
 }
 
 module "lambdas" {
-  source           = "../../modules/lambdas"
-  ProjectId        = var.ProjectId
-  RunEnvironment   = "prod"
-  LinkryKvArn      = aws_cloudfront_key_value_store.linkry_kv.arn
-  OriginVerifyKey  = random_password.origin_verify_key.result
-  LogRetentionDays = 30
-  EmailDomain      = var.EmailDomain
+  source                           = "../../modules/lambdas"
+  ProjectId                        = var.ProjectId
+  RunEnvironment                   = "prod"
+  LinkryKvArn                      = aws_cloudfront_key_value_store.linkry_kv.arn
+  CurrentOriginVerifyKey           = module.origin_verify.current_origin_verify_key
+  PreviousOriginVerifyKey          = module.origin_verify.previous_origin_verify_key
+  PreviousOriginVerifyKeyExpiresAt = module.origin_verify.previous_invalid_time
+  LogRetentionDays                 = 30
+  EmailDomain                      = var.EmailDomain
 }
 
 module "frontend" {
   source             = "../../modules/frontend"
   BucketPrefix       = local.bucket_prefix
   CoreLambdaHost     = module.lambdas.core_function_url
-  OriginVerifyKey    = random_password.origin_verify_key.result
+  OriginVerifyKey    = module.origin_verify.current_origin_verify_key
   ProjectId          = var.ProjectId
   CoreCertificateArn = var.CoreCertificateArn
   CorePublicDomain   = var.CorePublicDomain
+  CoreSlowLambdaHost = module.lambdas.core_slow_function_url
   IcalPublicDomain   = var.IcalPublicDomain
   LinkryPublicDomain = var.LinkryPublicDomain
   LinkryKvArn        = aws_cloudfront_key_value_store.linkry_kv.arn

terraform/envs/prod/main.tf

@@ -46,43 +46,39 @@ locals {
   }
 }
 
-
-module "lambda_warmer" {
-  source           = "github.com/acm-uiuc/terraform-modules/lambda-warmer?ref=v1.0.1"
-  function_to_warm = module.lambdas.core_api_lambda_name
-}
 module "dynamo" {
   source    = "../../modules/dynamo"
   ProjectId = var.ProjectId
 }
 
-resource "random_password" "origin_verify_key" {
-  length  = 16
-  special = false
-  keepers = {
-    force_recreation = formatdate("DD-MMM-YYYY", plantimestamp())
-  }
+module "origin_verify" {
+  source    = "../../modules/origin_verify"
+  ProjectId = var.ProjectId
 }
+
 resource "aws_cloudfront_key_value_store" "linkry_kv" {
   name = "${var.ProjectId}-cloudfront-linkry-kv"
 }
 
 
 module "lambdas" {
-  source           = "../../modules/lambdas"
-  ProjectId        = var.ProjectId
-  RunEnvironment   = "dev"
-  LinkryKvArn      = aws_cloudfront_key_value_store.linkry_kv.arn
-  OriginVerifyKey  = random_password.origin_verify_key.result
-  LogRetentionDays = 30
-  EmailDomain      = var.EmailDomain
+  source                           = "../../modules/lambdas"
+  ProjectId                        = var.ProjectId
+  RunEnvironment                   = "dev"
+  LinkryKvArn                      = aws_cloudfront_key_value_store.linkry_kv.arn
+  CurrentOriginVerifyKey           = module.origin_verify.current_origin_verify_key
+  PreviousOriginVerifyKey          = module.origin_verify.previous_origin_verify_key
+  PreviousOriginVerifyKeyExpiresAt = module.origin_verify.previous_invalid_time
+  LogRetentionDays                 = 30
+  EmailDomain                      = var.EmailDomain
 }
 
 module "frontend" {
   source             = "../../modules/frontend"
   BucketPrefix       = local.bucket_prefix
   CoreLambdaHost     = module.lambdas.core_function_url
-  OriginVerifyKey    = random_password.origin_verify_key.result
+  CoreSlowLambdaHost = module.lambdas.core_slow_function_url
+  OriginVerifyKey    = module.origin_verify.current_origin_verify_key
   ProjectId          = var.ProjectId
   CoreCertificateArn = var.CoreCertificateArn
   CorePublicDomain   = var.CorePublicDomain

terraform/envs/qa/.terraform.lock.hcl

@@ -25,8 +25,9 @@ resource "aws_cloudwatch_metric_alarm" "app_dlq_messages_alarm" {
 }
 
 resource "aws_cloudwatch_metric_alarm" "app_latency_alarm" {
-  alarm_name          = "${var.resource_prefix}-latency-high"
-  alarm_description   = "Trailing Mean - 95% API gateway latency is > 1.25s for 2 times in 4 minutes."
+  for_each            = var.performance_noreq_lambdas
+  alarm_name          = "${each.value}-latency-high"
+  alarm_description   = "${replace(each.value, var.resource_prefix, "")} Trailing Mean - 95% API gateway latency is > 1.25s for 2 times in 4 minutes."
   namespace           = "AWS/Lambda"
   metric_name         = "UrlRequestLatency"
   extended_statistic  = "tm95"
@@ -38,13 +39,14 @@ resource "aws_cloudwatch_metric_alarm" "app_latency_alarm" {
     var.standard_sns_arn
   ]
   dimensions = {
-    FunctionName = var.main_lambda_function_name
+    FunctionName = each.value
   }
 }
 
 resource "aws_cloudwatch_metric_alarm" "app_no_requests_alarm" {
-  alarm_name          = "${var.resource_prefix}-no-requests"
-  alarm_description   = "No requests have been received in the past 5 minutes."
+  for_each            = var.performance_noreq_lambdas
+  alarm_name          = "${each.value}-no-requests"
+  alarm_description   = "${replace(each.value, var.resource_prefix, "")}: no requests have been received in the past 5 minutes."
   namespace           = "AWS/Lambda"
   metric_name         = "UrlRequestCount"
   statistic           = "Sum"
@@ -56,13 +58,14 @@ resource "aws_cloudwatch_metric_alarm" "app_no_requests_alarm" {
     var.priority_sns_arn
   ]
   dimensions = {
-    FunctionName = var.main_lambda_function_name
+    FunctionName = each.value
   }
 }
 
 resource "aws_cloudwatch_metric_alarm" "app_invocation_error_alarm" {
-  alarm_name          = "${var.resource_prefix}-error-invocation"
-  alarm_description   = "Lambda threw an error, meaning the init of the application itself has encountered an error"
+  for_each            = var.all_lambdas
+  alarm_name          = "${each.value}-error-invocation"
+  alarm_description   = "${replace(each.value, var.resource_prefix, "")} lambda threw an error, meaning the init of the application itself has encountered an error"
   namespace           = "AWS/Lambda"
   metric_name         = "Errors"
   statistic           = "Sum"
@@ -74,7 +77,7 @@ resource "aws_cloudwatch_metric_alarm" "app_invocation_error_alarm" {
     var.priority_sns_arn
   ]
   dimensions = {
-    FunctionName = var.main_lambda_function_name
+    FunctionName = each.value
   }
 }