Skip to content

Commit 9335030

Browse files
cemms1cemms1
committed
WIP add scheduled validation lambda for ab testing expiry dates
1 parent 94786fd commit 9335030

22 files changed

+1344
-1
lines changed

‎.github/workflows/ab-testing-alerts.yml‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
name: 🧪 AB testing expiry checker
2+
3+
permissions:
4+
contents: read
5+
6+
on:
7+
schedule:
8+
# runs at 5am daily
9+
- cron: '0 5 * * *'
10+
11+
jobs:
12+
check-expiry:
13+
runs-on: ubuntu-latest
14+
name: Check test expiry
15+
defaults:
16+
run:
17+
working-directory: ab-testing/config
18+
env:
19+
FASTLY_AB_TESTING_CONFIG: ${{ secrets.FASTLY_PROD_AB_TESTING_CONFIG }}
20+
FASTLY_API_TOKEN: ${{ secrets.FASTLY_PROD_API_TOKEN }}
21+
22+
steps:
23+
- uses: actions/checkout@v5
24+
25+
- name: Set up Node environment
26+
uses: ./.github/actions/setup-node-env
27+
28+
- name: Validate
29+
run: pnpm validate
30+
31+
- name: Build
32+
run: pnpm build
33+
34+
- uses: actions/upload-artifact@v5
35+
with:
36+
name: ab-testing-build
37+
path: ab-testing/config/dist
38+
39+
lambda-ci:
40+
name: Lambda CI
41+
runs-on: ubuntu-latest
42+
defaults:
43+
run:
44+
working-directory: ab-testing/deploy-lambda
45+
permissions:
46+
contents: read
47+
steps:
48+
- uses: actions/checkout@v5
49+
50+
- name: Set up Node environment
51+
uses: ./.github/actions/setup-node-env
52+
53+
- name: Build Lambda
54+
run: pnpm build
55+
56+
- name: Zip app artifact
57+
run: |
58+
cd dist
59+
zip -r lambda.zip .
60+
zip -j lambda.zip ../package.json
61+
62+
- name: Save build
63+
uses: actions/upload-artifact@v5
64+
with:
65+
name: ab-testing-lambda-build
66+
path: ab-testing/deploy-lambda/dist/lambda.zip

‎ab-testing/cdk/lib/validationLambda.ts‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
import {
2+
GuStack,
3+
type GuStackProps,
4+
} from "@guardian/cdk/lib/constructs/core/stack.js";
5+
import { GuScheduledLambda } from "@guardian/cdk";
6+
import type { App } from "aws-cdk-lib";
7+
import { Runtime } from "aws-cdk-lib/aws-lambda";
8+
import { Schedule } from "aws-cdk-lib/aws-events";
9+
10+
export const lambdaFunctionName = "ab-testing-validation-lambda";
11+
12+
export class AbTestingValidationLambda extends GuStack {
13+
constructor(scope: App, id: string, props: GuStackProps) {
14+
super(scope, id, props);
15+
16+
const validationLambda = new GuScheduledLambda(
17+
this,
18+
"AbTestingValidationLambda",
19+
{
20+
app: lambdaFunctionName,
21+
fileName: "",
22+
handler: "",
23+
rules: [
24+
{
25+
schedule: Schedule.expression(
26+
"cron(0 8 ? * MON-FRI *)",
27+
),
28+
description: "Daily expiry checks for AB testing",
29+
},
30+
],
31+
monitoringConfiguration: { noMonitoring: true },
32+
runtime: Runtime.NODEJS_22_X,
33+
},
34+
);
35+
}
36+
}

‎ab-testing/config/abTests.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ const ABTests: ABTest[] = [
3737
description:
3838
"Tests whether we can get the users email, hash it and pass pd value to the userId array",
3939
owners: ["[email protected]"],
40-
expirationDate: `2026-01-15`,
40+
expirationDate: `2025-12-15`,
4141
type: "client",
4242
status: "OFF",
4343
audienceSize: 10 / 100,

‎ab-testing/config/scripts/validation/checkExpiry.test.ts‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
import assert from "node:assert";
2+
import test from "node:test";
3+
import type { ABTest } from "../../types.ts";
4+
import { checkExpiry } from "./checkExpiry.ts";
5+
6+
function getOffsetDate(days: number): ABTest["expirationDate"] {
7+
const today = new Date();
8+
today.setDate(today.getDate() + days);
9+
return today.toISOString().split("T")[0] as ABTest["expirationDate"]; // Format as YYYY-MM-DD
10+
}
11+
12+
test("checkExpiry - does not flag when the expiration is far in the future", () => {
13+
const futureDayTest: ABTest = {
14+
name: "commercial-future",
15+
description: "End on a weekday",
16+
owners: ["[email protected]"],
17+
status: "ON",
18+
expirationDate: getOffsetDate(10),
19+
type: "client",
20+
audienceSize: 10 / 100,
21+
groups: ["control", "variant"],
22+
};
23+
24+
assert.deepStrictEqual(checkExpiry([futureDayTest]), {
25+
within2Days: [],
26+
within1Day: [],
27+
expired: [],
28+
});
29+
});
30+
31+
test("checkExpiry - flags when the expiration is within two days", () => {
32+
const dayAfterTomorrowTest: ABTest = {
33+
name: "commercial-future",
34+
description: "End on a weekday",
35+
owners: ["[email protected]"],
36+
status: "ON",
37+
expirationDate: getOffsetDate(2),
38+
type: "client",
39+
audienceSize: 10 / 100,
40+
groups: ["control", "variant"],
41+
};
42+
43+
assert.deepStrictEqual(checkExpiry([dayAfterTomorrowTest]), {
44+
within2Days: [dayAfterTomorrowTest],
45+
within1Day: [],
46+
expired: [],
47+
});
48+
});
49+
50+
test("checkExpiry - flags when the expiration is within one day", () => {
51+
const tomorrowTest: ABTest = {
52+
name: "commercial-future",
53+
description: "End on a weekday",
54+
owners: ["[email protected]"],
55+
status: "ON",
56+
expirationDate: getOffsetDate(1),
57+
type: "client",
58+
audienceSize: 10 / 100,
59+
groups: ["control", "variant"],
60+
};
61+
62+
assert.deepStrictEqual(checkExpiry([tomorrowTest]), {
63+
within2Days: [],
64+
within1Day: [tomorrowTest],
65+
expired: [],
66+
});
67+
});
68+
69+
test("checkExpiry - marks test as expired with a date of today", () => {
70+
const todayTest: ABTest = {
71+
name: "commercial-future",
72+
description: "End on a weekday",
73+
owners: ["[email protected]"],
74+
status: "ON",
75+
expirationDate: getOffsetDate(0),
76+
type: "client",
77+
audienceSize: 10 / 100,
78+
groups: ["control", "variant"],
79+
};
80+
81+
assert.deepStrictEqual(checkExpiry([todayTest]), {
82+
within2Days: [],
83+
within1Day: [],
84+
expired: [todayTest],
85+
});
86+
});
87+
88+
test("checkExpiry - flags when the expiration is in the past", () => {
89+
const pastTest: ABTest = {
90+
name: "commercial-future",
91+
description: "End on a weekday",
92+
owners: ["[email protected]"],
93+
status: "ON",
94+
expirationDate: getOffsetDate(-2),
95+
type: "client",
96+
audienceSize: 10 / 100,
97+
groups: ["control", "variant"],
98+
};
99+
100+
assert.deepStrictEqual(checkExpiry([pastTest]), {
101+
within2Days: [],
102+
within1Day: [],
103+
expired: [pastTest],
104+
});
105+
});

‎ab-testing/config/scripts/validation/checkExpiry.ts‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
import type { ABTest } from "../../types.ts";
2+
3+
type Accumulator = {
4+
within2Days: ABTest[];
5+
within1Day: ABTest[];
6+
expired: ABTest[];
7+
};
8+
9+
export function checkExpiry(tests: ABTest[]) {
10+
return tests.reduce(
11+
(acc: Accumulator, test: ABTest) => {
12+
const expirationDate = new Date(test.expirationDate);
13+
const now = Date.now();
14+
const oneDay = 1000 * 60 * 60 * 24;
15+
16+
// Has the test expired?
17+
if (expirationDate < new Date(now)) {
18+
acc.expired.push(test);
19+
return acc;
20+
}
21+
22+
// Is the test expiring within the next day?
23+
if (expirationDate < new Date(now + oneDay)) {
24+
acc.within1Day.push(test);
25+
return acc;
26+
}
27+
28+
// Is the test expiring within the next two days?
29+
if (expirationDate < new Date(now + oneDay + oneDay)) {
30+
acc.within2Days.push(test);
31+
return acc;
32+
}
33+
34+
return acc;
35+
},
36+
{ within2Days: [], within1Day: [], expired: [] },
37+
);
38+
}

‎ab-testing/config/validation.handler.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
import { activeABtests } from "./abTests.ts";
2+
import { checkExpiry } from "./scripts/validation/checkExpiry.ts";
3+
4+
export const handler = async () => {
5+
const expiryChecks = checkExpiry(activeABtests);
6+
};

‎ab-testing/validation-lambda/README.md‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# @guardian/ab-testing-deploy-lambda
2+
3+
This directory contains the AWS Lambda function responsible for deploying AB testing configurations to Fastly.
4+
5+
The lambda is triggered during cloudformation to update the AB testing configurations stored in Fastly dictionaries. It reads the latest AB test definitions and their associated MVT IDs, then updates the Fastly service accordingly.
6+
7+
It is implemented to be invoked as a [`CustomResource`](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/template-custom-resources.html), which is a way to extend CloudFormation functionality by writing custom provisioning logic in an AWS Lambda.
8+
9+
## How it works
10+
11+
When the deployment lambda is invoked, it performs the following steps:
12+
13+
1. Reads the AB test definitions and MVT ID mappings from the artifacts located in S3.
14+
2. Connects to the Fastly API using credentials from AWS Secrets Manager.
15+
3. Updates the relevant Fastly dictionaries with the new AB test configurations.
16+
17+
## Local Development
18+
19+
Get `frontend` credentials from Janus.
20+
21+
install dependencies:
22+
23+
```bash
24+
pnpm install
25+
```
26+
27+
Call the `run.ts` script, (be sure to announce you're using the CODE stage in the semaphore chat channel):
28+
29+
```bash
30+
STAGE=CODE ARTIFACT_BUCKET_NAME=the-bucket node src/run.ts
31+
```

‎ab-testing/validation-lambda/package.json‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
{
2+
"name": "@guardian/ab-testing-deploy-lambda",
3+
"version": "1.0.0",
4+
"description": "Lambda that deploys AB test configuration to Fastly edge-dictionaries",
5+
"main": "index.ts",
6+
"type": "module",
7+
"scripts": {
8+
"build": "rollup -c rollup.config.js",
9+
"test": "node --test --experimental-test-module-mocks --test-reporter spec './**/*.test.ts'",
10+
"lint": "eslint .",
11+
"prettier:check": "prettier . --check --cache",
12+
"prettier:fix": "prettier . --write --cache"
13+
},
14+
"dependencies": {
15+
"@aws-sdk/client-s3": "3.931.0",
16+
"@aws-sdk/client-ssm": "3.621.0",
17+
"@guardian/ab-testing-config": "workspace:ab-testing-config",
18+
"superstruct": "2.0.2"
19+
},
20+
"devDependencies": {
21+
"@guardian/cdk": "62.0.1",
22+
"@guardian/eslint-config": "12.0.1",
23+
"@guardian/tsconfig": "1.0.1",
24+
"@rollup/plugin-commonjs": "29.0.0",
25+
"@rollup/plugin-json": "6.1.0",
26+
"@rollup/plugin-node-resolve": "16.0.3",
27+
"@types/aws-lambda": "8.10.158",
28+
"@types/node": "22.17.0",
29+
"aws-cdk-lib": "2.220.0",
30+
"aws-lambda": "1.0.7",
31+
"esbuild": "0.27.0",
32+
"eslint": "9.39.1",
33+
"prettier": "3.0.3",
34+
"rollup": "4.53.2",
35+
"rollup-plugin-esbuild": "6.2.1",
36+
"type-fest": "4.21.0",
37+
"typescript": "5.9.3"
38+
}
39+
}

‎ab-testing/validation-lambda/rollup.config.js‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
import commonjs from "@rollup/plugin-commonjs";
2+
import json from "@rollup/plugin-json";
3+
import { nodeResolve } from "@rollup/plugin-node-resolve";
4+
import esbuild from "rollup-plugin-esbuild";
5+
6+
/** @type {import('rollup').RollupOptions} */
7+
const rollupConfig = {
8+
input: `src/index.ts`,
9+
output: {
10+
dir: "dist",
11+
format: "module",
12+
preserveModules: true,
13+
preserveModulesRoot: "src",
14+
sourcemap: true,
15+
entryFileNames: (chunkInfo) => {
16+
if (chunkInfo.name.includes("node_modules")) {
17+
// Simplify node_modules paths, if it's in node_modules without a package.json
18+
// it'll be assumed to be a commonjs module, which is incorrect and causes issues.
19+
return (
20+
chunkInfo.name.replace(
21+
/node_modules\/\.pnpm\/.*\/node_modules/,
22+
"external",
23+
) + ".js"
24+
);
25+
}
26+
27+
return "[name].js";
28+
},
29+
},
30+
external: ["@aws-sdk/*"],
31+
plugins: [
32+
commonjs(),
33+
nodeResolve({
34+
preferBuiltins: true,
35+
exportConditions: ["node"],
36+
resolveOnly: (moduleName) => !moduleName.startsWith("@aws-sdk"),
37+
}),
38+
json(),
39+
esbuild({
40+
include: /\.[jt]s?$/,
41+
}),
42+
],
43+
};
44+
45+
export default rollupConfig;

0 commit comments

Comments
 (0)