Merge pull request #14333 from guardian/jlk/new-ab-testing-definitions

Fastly Dictionary AB testing configuration and associated scripts

Author: GHaberis

.github/workflows/ab-testing-checks.yml

@@ -0,0 +1,38 @@
+name: Checks
+
+permissions:
+  contents: read
+
+on:
+  workflow_call:
+    secrets:
+      FASTLY_AB_TESTING_CONFIG:
+        required: true
+      FASTLY_API_TOKEN:
+        required: true
+
+jobs:
+  checks:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ab-testing
+    env:
+      FASTLY_AB_TESTING_CONFIG: ${{ secrets.FASTLY_AB_TESTING_CONFIG }}
+      FASTLY_API_TOKEN: ${{ secrets.FASTLY_API_TOKEN }}
+    steps:
+      - uses: actions/checkout@v4
+
+      # https://github.com/denoland/setup-deno#latest-stable-for-a-major
+      - uses: denoland/setup-deno@v1
+        with:
+          deno-version: v2.3
+
+      - name: Test
+        run: deno test
+
+      - name: Validate
+        run: deno run scripts/validation/index.ts
+
+      - name: Build
+        run: deno task build

.github/workflows/ab-testing-ci.yml

@@ -0,0 +1,16 @@
+name: 🧪 AB testing
+
+permissions:
+  contents: read
+
+on:
+  pull_request:
+    paths:
+      - 'ab-testing/**'
+
+jobs:
+  checks:
+    uses: ./.github/workflows/ab-testing-checks.yml
+    secrets:
+      FASTLY_AB_TESTING_CONFIG: ${{ secrets.FASTLY_CODE_AB_TESTING_CONFIG }}
+      FASTLY_API_TOKEN: ${{ secrets.FASTLY_CODE_API_TOKEN }}

.github/workflows/ab-testing-deploy-code.yml

@@ -0,0 +1,30 @@
+name: 🧪 Deploy CODE AB Testing Config
+
+permissions:
+  contents: read
+
+on:
+  workflow_dispatch:
+
+jobs:
+  ci:
+    uses: ./.github/workflows/ab-testing-checks.yml
+    secrets:
+      FASTLY_AB_TESTING_CONFIG: ${{ secrets.FASTLY_CODE_AB_TESTING_CONFIG }}
+      FASTLY_API_TOKEN: ${{ secrets.FASTLY_CODE_API_TOKEN }}
+
+  deploy:
+    name: Deploy CODE
+    needs: ci
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ab-testing
+    steps:
+      - name: Deploy
+        uses: ./.github/workflows/ab-testing-deploy.yml
+        with:
+          stage: code
+        env:
+          FASTLY_AB_TESTING_CONFIG: ${{ secrets.FASTLY_CODE_AB_TESTING_CONFIG }}
+          FASTLY_API_TOKEN: ${{ secrets.FASTLY_CODE_API_TOKEN }}

.github/workflows/ab-testing-deploy-prod.yml

@@ -0,0 +1,34 @@
+name: 🧪 Deploy PROD AB Testing Config
+
+permissions:
+  contents: read
+
+on:
+  workflow_dispatch:
+  # push:
+  #   branches:
+  #     - main
+  #   paths:
+  #     - 'ab-testing/**'
+
+jobs:
+  ci:
+    uses: ./.github/workflows/ab-testing-checks.yml
+    secrets:
+      FASTLY_AB_TESTING_CONFIG: ${{ secrets.FASTLY_PROD_AB_TESTING_CONFIG }}
+      FASTLY_API_TOKEN: ${{ secrets.FASTLY_PROD_API_TOKEN }}
+  deploy:
+    name: Deploy PROD
+    needs: ci
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ab-testing
+    steps:
+      - name: Deploy
+        uses: ./.github/workflows/ab-testing-deploy.yml
+        with:
+          stage: prod
+        env:
+          FASTLY_AB_TESTING_CONFIG: ${{ secrets.FASTLY_PROD_AB_TESTING_CONFIG }}
+          FASTLY_API_TOKEN: ${{ secrets.FASTLY_PROD_API_TOKEN }}

.github/workflows/ab-testing-deploy.yml

@@ -0,0 +1,35 @@
+name: 🧪 Deploy AB Testing Config
+
+permissions:
+  contents: read
+
+on:
+  workflow_call:
+    inputs:
+      stage:
+        description: 'Stage to deploy to (e.g., code, prod)'
+        required: true
+        type: string
+    secrets:
+      FASTLY_AB_TESTING_CONFIG:
+        required: true
+      FASTLY_API_TOKEN:
+        required: true
+
+jobs:
+  deploy:
+    name: Deploy
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ab-testing
+    steps:
+      - uses: actions/checkout@v4
+
+      # https://github.com/denoland/setup-deno#latest-stable-for-a-major
+      - uses: denoland/setup-deno@v1
+        with:
+          deno-version: v2.3
+
+      - name: Deploy
+        run: deno run deploy

.vscode/settings.json.required

@@ -2,7 +2,7 @@
 // Copy these to .vscode/settings.json to apply them to your editor – but don't commit it!
 // https://github.com/guardian/recommendations/blob/main/VSCode.md#do-not-commit-vscode
 {
-	"deno.enablePaths": ["scripts/deno"],
+	"deno.enablePaths": ["scripts/deno", "ab-testing/scripts"],
 	"deno.lint": true,
 	"deno.unstable": false,
 	"typescript.tsdk": "node_modules/typescript/lib",

ab-testing/README.md

@@ -0,0 +1,76 @@
+# AB Testing
+
+This directory contains the code and configuration for the AB testing framework used on theguardian.com.
+
+The [`abTest.ts`](./abTest.ts) module is where you should define your AB tests.
+
+Add your AB tests to the `abTests` array in the `abTest.ts` file. Each test should have a unique name.
+
+```ts
+{
+	name: 'webex-example-test',
+	description:
+		'Test something interesting on the site',
+	owners: ['[email protected]'],
+	status: 'ON',
+	expirationDate: '2050-12-30',
+	type: 'client',
+	audienceSize: 10 / 100,
+	audienceSpace: 'A',
+	groups: ['control', 'variant'],
+}
+```
+
+When you create a PR that modifies the `abTest.ts` file, a git hook and CI will run checks to ensure that your AB test is valid (not expired, enough space for the test etc.).
+
+When your PR is merged, the AB test will be automatically deployed to Fastly and be available at the same time as your changes.
+
+## Guidelines for AB Tests
+
+### Naming Conventions
+
+AB tests should be prefixed with the team associated with the test, for example `webex-example-test`. This helps to identify the team responsible for the test and is enforce by typescript validation.
+
+### Test Size and Groups
+
+The `audienceSize` is the size of the whole test and is divided between the test groups that you specify. The "resolution" of sizing is down to 0.1%, so groups will be rounded to the nearest 0.1%.
+
+Convention is to have groups named control and variant, but you can name them as you wish.
+
+A single group is also possible, for example if you're rolling out a new feature and don't need a control.
+
+### Client vs Server Side Tests
+
+All requests are processed by Fastly at the edge, however, ab testing of server-side logic in Frontend or DCR will need to be cached separately. Client side tests do not need to be cached separately, as they are applied in the browser after the response is delivered.
+
+Ensure that the `type` field is set to either `client` or `server` to indicate the type of test so that server side tests can be cached correctly, and client side tests are not splitting the cache unnecessarily.
+
+There's a limit of the number of concurrent server-side tests that can be run, enforce by the validation script, so it's important to use client-side tests where possible.
+
+### Test Expiration
+
+AB tests should have an expiration date set in the future. This is to ensure that tests do not run indefinitely.
+
+Expired tests will cause the ab testing validation to fail, and will not be deployed.
+
+Tests that expire while they are are in-flight will not be served by fastly, and should be removed from the `abTest.ts` file as soon as possible.
+
+### Audience Spaces
+
+Ideally AB tests would never overlap (users being in multiple tests), but sometimes this is unavoidable, for example when running a very large 50+% test without interrupting existing tests.
+
+To add a test where there is not enough space in the default audience space (`A`), you can specify a different `audienceSpace` in the test definition.
+
+For example if there are already 3 25% tests in space `A` totalling 75%, and you want to run a 50% test, you can set the `audienceSpace` to `B` to allow this test to overlap with the existing tests.
+
+## How it works
+
+The AB testing framework uses Deno to run scripts that validate and deploy the tests. The `deno.json` file contains the tasks that can be run, such as `validate`, `deploy`, and `build`.
+
+Which tests using which mvt ids is computed by the `build` task, which generates the `dist/mvts.json` file. This file contains the mapping of AB tests to MVT IDs.
+
+The algorithm allocates tests available MVT IDs based on the audience size and space. Allocation is deterministic, so the same test (with the same name) will always be assigned the same MVT ID as long as the test hasn't been removed, allowing for consistent user experience when tests are added/updated/removed. This also means that new/update tests will not use contiguous MVT IDs, but will instead use whichever MVT IDs are available.
+
+However, the allocation is completely separate for each audience space, so if you have a test in space `A` and move it to space `B`, it will be allocated different MVT IDs.
+
+The state of the AB tests is stored in Fastly dictionaries, which are updated when the `deploy` task is run. Logic in fastly VCL will then use these dictionaries to determine which users are in which test groups and set appropriate headers and/or cookies.

ab-testing/abTest.ts

@@ -45,20 +45,4 @@ export const ABTests: ABTest[] = [
 		groups: ['control', 'variant'],
 		shouldForceMetricsCollection: true,
 	},
-	{
-		name: 'commercial-large-overlap-test',
-		description:
-			'Allows viewing the beta version of the Europe network front',
-		owners: [
-			'[email protected]',
-			'[email protected]',
-		],
-		status: 'ON',
-		expirationDate: '2050-12-30',
-		type: 'server',
-		audienceSize: 50 / 100,
-		groups: ['control', 'variant'],
-		audienceSpace: 'B',
-		shouldForceMetricsCollection: true,
-	},
 ];

ab-testing/deno.json

@@ -0,0 +1,11 @@
+{
+	"tasks": {
+		"validate": "deno run scripts/validation/index.ts",
+		"deploy": "deno run --allow-read=dist --allow-env=FASTLY_AB_TESTING_CONFIG --allow-net=api.fastly.com:443 scripts/deploy/index.ts --mvts=dist/mvts.json --ab-tests=dist/ab-tests.json",
+		"build": "deno run --allow-write=dist --allow-env=FASTLY_AB_TESTING_CONFIG,FASTLY_API_TOKEN --allow-net=api.fastly.com:443 scripts/build/index.ts --mvts=dist/mvts.json --ab-tests=dist/ab-tests.json"
+	},
+	"imports": {
+		"@std/assert": "jsr:@std/assert@^1.0.13",
+		"@superstruct/core": "jsr:@superstruct/[email protected]"
+	}
+}