improve logging and monitoring of notification lambda

Author: cemms1

ab-testing/cdk/lib/__snapshots__/notificationLambda.test.ts.snap

@@ -309,11 +309,11 @@ exports[`The AB testing notification lambda stack > matches the CODE snapshot 1`
         },
         "Environment": {
           "Variables": {
+            "STAGE": "CODE",
             "EMAIL_DOMAIN": {
               "Ref": "EmailIdentityAbtestingnotificationlambdaE053C648"
             },
             "STACK": "frontend",
-            "STAGE": "CODE",
             "APP": "ab-testing-notification-lambda"
           }
         },
@@ -389,7 +389,7 @@ exports[`The AB testing notification lambda stack > matches the CODE snapshot 1`
               {
                 "Ref": "AbTestingNotificationLambda36F153E8"
               },
-              " exceeded 1% error rate"
+              " exceeded 0% error rate"
             ]
           ]
         },
@@ -483,7 +483,7 @@ exports[`The AB testing notification lambda stack > matches the CODE snapshot 1`
             "Value": "CODE"
           }
         ],
-        "Threshold": 1,
+        "Threshold": 0,
         "TreatMissingData": "notBreaching"
       }
     }
@@ -630,6 +630,16 @@ exports[`The AB testing notification lambda stack > matches the PROD snapshot 1`
         ]
       }
     },
+    "ABTestingNotificationErrors7CB687D5": {
+      "Type": "AWS::SNS::Subscription",
+      "Properties": {
+        "Endpoint": "[email protected]",
+        "Protocol": "email",
+        "TopicArn": {
+          "Ref": "AbTestingNotificationSnsTopicB3559144"
+        }
+      }
+    },
     "AbTestingNotificationLambdaServiceRole529D80CF": {
       "Type": "AWS::IAM::Role",
       "Properties": {
@@ -809,11 +819,11 @@ exports[`The AB testing notification lambda stack > matches the PROD snapshot 1`
         },
         "Environment": {
           "Variables": {
+            "STAGE": "PROD",
             "EMAIL_DOMAIN": {
               "Ref": "EmailIdentityAbtestingnotificationlambdaE053C648"
             },
             "STACK": "frontend",
-            "STAGE": "PROD",
             "APP": "ab-testing-notification-lambda"
           }
         },
@@ -949,7 +959,7 @@ exports[`The AB testing notification lambda stack > matches the PROD snapshot 1`
               {
                 "Ref": "AbTestingNotificationLambda36F153E8"
               },
-              " exceeded 1% error rate"
+              " exceeded 0% error rate"
             ]
           ]
         },
@@ -1043,7 +1053,7 @@ exports[`The AB testing notification lambda stack > matches the PROD snapshot 1`
             "Value": "PROD"
           }
         ],
-        "Threshold": 1,
+        "Threshold": 0,
         "TreatMissingData": "notBreaching"
       }
     }

ab-testing/cdk/lib/notificationLambda.ts

@@ -8,7 +8,7 @@ import type { App } from "aws-cdk-lib";
 import { Schedule } from "aws-cdk-lib/aws-events";
 import { Effect, PolicyStatement } from "aws-cdk-lib/aws-iam";
 import { Runtime } from "aws-cdk-lib/aws-lambda";
-import { Topic } from "aws-cdk-lib/aws-sns";
+import { Subscription, SubscriptionProtocol, Topic } from "aws-cdk-lib/aws-sns";
 
 const lambdaFunctionName = "ab-testing-notification-lambda";
 
@@ -39,6 +39,16 @@ export class AbTestingNotificationLambda extends GuStack {
 
 		const snsTopic = new Topic(this, "AbTestingNotificationSnsTopic");
 
+		// Notify teams in PROD if this lambda errors
+		if (props.stage === "PROD") {
+			new Subscription(this, "ABTestingNotificationErrors", {
+				topic: snsTopic,
+				// TODO: Change this to [email protected] after testing for a while
+				endpoint: "[email protected]",
+				protocol: SubscriptionProtocol.EMAIL,
+			});
+		}
+
 		const lambda = new GuScheduledLambda(
 			this,
 			"AbTestingNotificationLambda",
@@ -49,10 +59,11 @@ export class AbTestingNotificationLambda extends GuStack {
 				rules: this.stage === "PROD" ? [runDailyRule] : [],
 				monitoringConfiguration: {
 					snsTopicName: snsTopic.topicName,
-					toleratedErrorPercentage: 1,
+					toleratedErrorPercentage: 0,
 				},
 				runtime: Runtime.NODEJS_22_X,
 				environment: {
+					STAGE: props.stage,
 					EMAIL_DOMAIN: emailIdentity.emailIdentityName,
 				},
 			},

ab-testing/notification-lambda/src/index.ts

@@ -6,6 +6,14 @@ export const handler = async (): Promise<void> => {
 	const expiringAbTestsByOwner =
 		getExpiringAbTestsGroupedByOwner(activeABtests);
 
+	// Check if any test owners have expiring tests
+	// Early return if there are no results
+	if (!Object.keys(expiringAbTestsByOwner).length) {
+		console.log("No owners found with expiring tests");
+		Promise.resolve();
+	}
+
+	// Sending emails to owners with expiring tests
 	await Promise.all(
 		Object.entries(expiringAbTestsByOwner).map(([owner, abTests]) =>
 			sendEmail(owner, abTests),

ab-testing/notification-lambda/src/lib/email.ts

@@ -129,6 +129,11 @@ export const sendEmail = async (
 					Source: `AB Testing <notifications@${process.env.EMAIL_DOMAIN}>`,
 					Destination: {
 						ToAddresses: [recipient],
+						CcAddresses:
+							// Only add CC addresses in prod
+							process.env.STAGE === "PROD"
+								? ["[email protected]"]
+								: [],
 					},
 					Message: {
 						Subject: {
@@ -150,10 +155,15 @@ export const sendEmail = async (
 			)
 			.then(() =>
 				console.log(
-					`Sent AB test expiry reminder email to ${recipient}`,
+					`Sent AB test expiry reminder email to ${recipient} for tests
+					${Object.values(expiringAbTests)
+						.flat()
+						.map((test) => test.name)
+						.join(", ")}`,
 				),
 			);
 	} catch (error) {
-		console.log(`Error sending email to ${recipient}`, error);
+		console.error(`Error sending email to ${recipient}`, error);
+		throw error;
 	}
 };

ab-testing/notification-lambda/src/run.ts

@@ -11,10 +11,11 @@
 
 import { handler } from "./index.ts";
 
+process.env.STAGE = `LOCAL`;
 process.env.EMAIL_DOMAIN = `abtesting.code.dev-gutools.co.uk`;
 
-console.log(`Running lambda handler locally\n`);
-console.log(`EMAIL_DOMAIN=${process.env.EMAIL_DOMAIN}\n`);
+console.debug(`Running lambda handler locally\n`);
+console.debug(`EMAIL_DOMAIN=${process.env.EMAIL_DOMAIN}\n`);
 
 void (async () => {
 	try {