integration tests

Author: ploomiz

.gitignore

@@ -6,3 +6,5 @@ logs/
 temp
 .env
 yarn-error.log
+
+test/assignmentTestData

jest.config.js

@@ -9,6 +9,7 @@ module.exports = {
   transform: {
     '^.+\\.(t|j)s$': 'ts-jest',
   },
+  globalSetup: './test/globalSetup.ts',
   collectCoverageFrom: ['**/*.(t|j)s'],
   coverageDirectory: 'coverage/',
   testEnvironment: 'node',

package.json

@@ -29,6 +29,7 @@
   },
   "homepage": "https://github.com/Eppo-exp/js-client-sdk#readme",
   "devDependencies": {
+    "@google-cloud/storage": "^6.1.0",
     "@microsoft/api-documenter": "^7.17.17",
     "@microsoft/api-extractor": "^7.25.0",
     "@types/jest": "^28.1.1",
@@ -44,8 +45,10 @@
     "jest": "^28.1.1",
     "jest-environment-jsdom": "^28.1.1",
     "prettier": "^2.7.1",
+    "testdouble": "^3.16.6",
     "ts-jest": "^28.0.5",
-    "typescript": "^4.7.3"
+    "typescript": "^4.7.3",
+    "xhr-mock": "^2.5.1"
   },
   "dependencies": {
     "axios": "^0.27.2"

src/eppo-client.spec.ts

@@ -0,0 +1,188 @@
+/**
+ * @jest-environment jsdom
+ */
+import * as td from 'testdouble';
+import mock from 'xhr-mock';
+
+import { IAssignmentTestCase, readAssignmentTestData } from '../test/testHelpers';
+
+import EppoClient from './eppo-client';
+import { IExperimentConfiguration } from './experiment/experiment-configuration';
+import ExperimentConfigurationRequestor from './experiment/experiment-configuration-requestor';
+import { IVariation } from './experiment/variation';
+import { OperatorType } from './rule';
+
+import { init } from '.';
+
+describe('EppoClient E2E test', () => {
+  beforeAll(() => {
+    window.sessionStorage.clear();
+    mock.setup();
+    mock.get(/randomized_assignment\/config*/, (_req, res) => {
+      const testCases: IAssignmentTestCase[] = readAssignmentTestData();
+      const assignmentConfig: Record<string, IExperimentConfiguration> = {};
+      testCases.forEach(({ experiment, percentExposure, variations }) => {
+        assignmentConfig[experiment] = {
+          name: experiment,
+          percentExposure,
+          enabled: true,
+          subjectShards: 10000,
+          variations,
+          overrides: {},
+          rules: [],
+        };
+      });
+      return res.status(200).body(JSON.stringify({ experiments: assignmentConfig }));
+    });
+  });
+
+  afterAll(() => {
+    mock.teardown();
+  });
+
+  describe('getAssignment', () => {
+    it.each(readAssignmentTestData())(
+      'test variation assignment splits',
+      async ({
+        variations,
+        experiment,
+        percentExposure,
+        subjects,
+        expectedAssignments,
+      }: IAssignmentTestCase) => {
+        console.log(`---- Test Case for ${experiment} Experiment ----`);
+        const assignments = await getAssignments(subjects, experiment);
+        // verify the assingments don't change across test runs (deterministic)
+        expect(assignments).toEqual(expectedAssignments);
+        const expectedVariationSplitPercentage = percentExposure / variations.length;
+        const unassignedCount = assignments.filter((assignment) => assignment == null).length;
+        expectToBeCloseToPercentage(unassignedCount / assignments.length, 1 - percentExposure);
+        variations.forEach((variation) => {
+          validateAssignmentCounts(assignments, expectedVariationSplitPercentage, variation);
+        });
+      },
+    );
+  });
+
+  it('returns subject from overrides', () => {
+    const mockConfigRequestor = td.object<ExperimentConfigurationRequestor>();
+    const experiment = 'experiment_5';
+    td.when(mockConfigRequestor.getConfiguration(experiment)).thenReturn({
+      name: experiment,
+      percentExposure: 1,
+      enabled: true,
+      subjectShards: 100,
+      variations: [
+        {
+          name: 'control',
+          shardRange: {
+            start: 0,
+            end: 33,
+          },
+        },
+        {
+          name: 'variant-1',
+          shardRange: {
+            start: 34,
+            end: 66,
+          },
+        },
+        {
+          name: 'variant-2',
+          shardRange: {
+            start: 67,
+            end: 100,
+          },
+        },
+      ],
+      overrides: {
+        a90ea45116d251a43da56e03d3dd7275: 'variant-2',
+      },
+    });
+    const client = new EppoClient('subject-1', mockConfigRequestor);
+    const assignment = client.getAssignment(experiment);
+    expect(assignment).toEqual('variant-2');
+  });
+
+  it('only returns variation if subject matches rules', () => {
+    const mockConfigRequestor = td.object<ExperimentConfigurationRequestor>();
+    const experiment = 'experiment_5';
+    td.when(mockConfigRequestor.getConfiguration(experiment)).thenReturn({
+      name: experiment,
+      percentExposure: 1,
+      enabled: true,
+      subjectShards: 100,
+      variations: [
+        {
+          name: 'control',
+          shardRange: {
+            start: 0,
+            end: 50,
+          },
+        },
+        {
+          name: 'treatment',
+          shardRange: {
+            start: 50,
+            end: 100,
+          },
+        },
+      ],
+      overrides: {},
+      rules: [
+        {
+          conditions: [
+            {
+              operator: OperatorType.GT,
+              attribute: 'appVersion',
+              value: 10,
+            },
+          ],
+        },
+      ],
+    });
+    let client = new EppoClient('subject-1', mockConfigRequestor, { appVersion: 9 });
+    let assignment = client.getAssignment(experiment);
+    expect(assignment).toEqual(null);
+    client = new EppoClient('subject-1', mockConfigRequestor);
+    assignment = client.getAssignment(experiment);
+    expect(assignment).toEqual(null);
+    client = new EppoClient('subject-1', mockConfigRequestor, { appVersion: 11 });
+    assignment = client.getAssignment(experiment);
+    expect(assignment).toEqual('control');
+  });
+
+  function validateAssignmentCounts(
+    assignments: string[],
+    expectedPercentage: number,
+    variation: IVariation,
+  ) {
+    const assignedCount = assignments.filter((assignment) => assignment === variation.name).length;
+    console.log(
+      `Expect variation ${variation.name} percentage of ${
+        assignedCount / assignments.length
+      } to be close to ${expectedPercentage}`,
+    );
+    expectToBeCloseToPercentage(assignedCount / assignments.length, expectedPercentage);
+  }
+
+  // expect assignment count to be within 5 percentage points of the expected count (because the hash output is random)
+  function expectToBeCloseToPercentage(percentage: number, expectedPercentage: number) {
+    expect(percentage).toBeGreaterThanOrEqual(expectedPercentage - 0.05);
+    expect(percentage).toBeLessThanOrEqual(expectedPercentage + 0.05);
+  }
+
+  async function getAssignments(subjects: string[], experiment: string): Promise<string[]> {
+    const assignments: string[] = [];
+    for (const subjectKey of subjects) {
+      const client = await init({
+        apiKey: 'dummy',
+        baseUrl: 'http://127.0.0.1:4000',
+        subjectKey,
+      });
+      const assignment = client.getAssignment(experiment);
+      assignments.push(assignment);
+    }
+    return assignments;
+  }
+});

test/globalSetup.ts

@@ -0,0 +1,25 @@
+import * as fs from 'fs';
+
+import { Storage } from '@google-cloud/storage';
+
+import { TEST_DATA_DIR } from './testHelpers';
+
+const storage = new Storage();
+
+async function downloadTestDataFiles() {
+  const [files] = await storage.bucket('sdk-test-data').getFiles({
+    prefix: 'assignment/test-case',
+  });
+  return Promise.all(
+    files.map((file, index) => {
+      return file.download({ destination: `${TEST_DATA_DIR}test-case-${index}.json` });
+    }),
+  );
+}
+
+export default async () => {
+  if (!fs.existsSync(TEST_DATA_DIR)) {
+    fs.mkdirSync(TEST_DATA_DIR);
+    await downloadTestDataFiles();
+  }
+};

test/testHelpers.ts

@@ -0,0 +1,24 @@
+import * as fs from 'fs';
+
+import { IVariation } from '../src/experiment/variation';
+
+export const TEST_DATA_DIR = './test/assignmentTestData/';
+
+export interface IAssignmentTestCase {
+  experiment: string;
+  percentExposure: number;
+  variations: IVariation[];
+  subjects: string[];
+  expectedAssignments: string[];
+}
+
+export function readAssignmentTestData(): IAssignmentTestCase[] {
+  const testDataDir = './test/assignmentTestData/';
+  const testCaseData: IAssignmentTestCase[] = [];
+  const testCaseFiles = fs.readdirSync(testDataDir);
+  testCaseFiles.forEach((file) => {
+    const testCase = JSON.parse(fs.readFileSync(testDataDir + file, 'utf8'));
+    testCaseData.push(testCase);
+  });
+  return testCaseData;
+}